X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c6ec580a9c79d1d21cffa6454d871538ba7c14f0..b8be976adfabb17718dee8c569129cdf28036380:/sdk/go/arvadosclient/arvadosclient.go

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index efe1583297..8cdfa484bd 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -14,6 +14,7 @@ import (
 	"os"
 	"regexp"
 	"strings"
+	"time"
 )
 
 type StringMatcher func(string) bool
@@ -25,6 +26,14 @@ var MissingArvadosApiHost = errors.New("Missing required environment variable AR
 var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
 var ErrInvalidArgument = errors.New("Invalid argument")
 
+// A common failure mode is to reuse a keepalive connection that has been
+// terminated (in a way that we can't detect) for being idle too long.
+// POST and DELETE are not safe to retry automatically, so we minimize
+// such failures by always using a new or recently active socket.
+var MaxIdleConnectionDuration = 30 * time.Second
+
+var RetryDelay = 2 * time.Second
+
 // Indicates an error that was returned by the API server.
 type APIServerError struct {
 	// Address of server returning error, of the form "host:port".
@@ -80,6 +89,8 @@ type ArvadosClient struct {
 	// Discovery document
 	DiscoveryDoc Dict
 
+	lastClosedIdlesAt time.Time
+
 	// Number of retries
 	Retries int
 }
@@ -109,6 +120,8 @@ func MakeArvadosClient() (ac ArvadosClient, err error) {
 		return ac, MissingArvadosApiToken
 	}
 
+	ac.lastClosedIdlesAt = time.Now()
+
 	return ac, err
 }
 
@@ -150,14 +163,26 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
 		}
 	}
 
+	retryable := false
+	switch method {
+	case "GET", "HEAD", "PUT", "OPTIONS", "DELETE":
+		retryable = true
+	}
+
+	// Non-retryable methods such as POST are not safe to retry automatically,
+	// so we minimize such failures by always using a new or recently active socket
+	if !retryable {
+		if time.Since(c.lastClosedIdlesAt) > MaxIdleConnectionDuration {
+			c.lastClosedIdlesAt = time.Now()
+			c.Client.Transport.(*http.Transport).CloseIdleConnections()
+		}
+	}
+
 	// Make the request
-	remainingTries := 1 + c.Retries
 	var req *http.Request
 	var resp *http.Response
-	var errs []string
-	var badResp bool
 
-	for remainingTries > 0 {
+	for attempt := 0; attempt <= c.Retries; attempt++ {
 		if method == "GET" || method == "HEAD" {
 			u.RawQuery = vals.Encode()
 			if req, err = http.NewRequest(method, u.String(), nil); err != nil {
@@ -178,10 +203,8 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
 
 		resp, err = c.Client.Do(req)
 		if err != nil {
-			if method == "GET" || method == "HEAD" || method == "PUT" {
-				errs = append(errs, err.Error())
-				badResp = false
-				remainingTries -= 1
+			if retryable {
+				time.Sleep(RetryDelay)
 				continue
 			} else {
 				return nil, err
@@ -194,27 +217,19 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
 
 		defer resp.Body.Close()
 
-		if resp.StatusCode == 408 ||
-			resp.StatusCode == 409 ||
-			resp.StatusCode == 422 ||
-			resp.StatusCode == 423 ||
-			resp.StatusCode == 500 ||
-			resp.StatusCode == 502 ||
-			resp.StatusCode == 503 ||
-			resp.StatusCode == 504 {
-			badResp = true
-			remainingTries -= 1
+		switch resp.StatusCode {
+		case 408, 409, 422, 423, 500, 502, 503, 504:
+			time.Sleep(RetryDelay)
 			continue
-		} else {
+		default:
 			return nil, newAPIServerError(c.ApiServer, resp)
 		}
 	}
 
-	if badResp {
+	if resp != nil {
 		return nil, newAPIServerError(c.ApiServer, resp)
-	} else {
-		return nil, fmt.Errorf("%v", errs)
 	}
+	return nil, err
 }
 
 func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError {
@@ -258,7 +273,7 @@ func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError
 // Returns a non-nil error if an error occurs making the API call, the
 // API responds with a non-successful HTTP status, or an error occurs
 // parsing the response body.
-func (c ArvadosClient) Call(method string, resourceType string, uuid string, action string, parameters Dict, output interface{}) error {
+func (c ArvadosClient) Call(method, resourceType, uuid, action string, parameters Dict, output interface{}) error {
 	reader, err := c.CallRaw(method, resourceType, uuid, action, parameters)
 	if reader != nil {
 		defer reader.Close()