5538: add test with a connection idle for longer than MaxIdleConnectionDuration

[arvados.git] / sdk / go / arvadosclient / arvadosclient.go

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 1cce0a7fc92d24e21fa694add86c75c63952eb46..18e1074bf6f6c801c21593bc57586a902807f8b4 100644 (file)
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -14,6 +14,7 @@ import (
        "os"
        "regexp"
        "strings"
+       "time"
 )
 
 type StringMatcher func(string) bool
@@ -25,6 +26,12 @@ var MissingArvadosApiHost = errors.New("Missing required environment variable AR
 var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
 var ErrInvalidArgument = errors.New("Invalid argument")
 
+// A common failure mode is to reuse a keepalive connection that has been
+// terminated (in a way that we can't detect) for being idle too long.
+// POST and DELETE are not safe to retry automatically, so we minimize
+// such failures by always using a new or recently active socket.
+var MaxIdleConnectionDuration = 30 * time.Second
+
 // Indicates an error that was returned by the API server.
 type APIServerError struct {
        // Address of server returning error, of the form "host:port".
@@ -76,6 +83,8 @@ type ArvadosClient struct {
 
        // Discovery document
        DiscoveryDoc Dict
+
+       lastClosedIdlesAt time.Time
 }
 
 // Create a new ArvadosClient, initialized with standard Arvados environment
@@ -101,6 +110,8 @@ func MakeArvadosClient() (ac ArvadosClient, err error) {
                return ac, MissingArvadosApiToken
        }
 
+       ac.lastClosedIdlesAt = time.Now()
+
        return ac, err
 }
 
@@ -158,6 +169,15 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
                req.Header.Add("X-External-Client", "1")
        }
 
+       // POST and DELETE are not safe to retry automatically, so we minimize
+       // such failures by always using a new or recently active socket
+       if method == "POST" || method == "DELETE" {
+               if time.Since(c.lastClosedIdlesAt) > MaxIdleConnectionDuration {
+                       c.lastClosedIdlesAt = time.Now()
+                       c.Client.Transport.(*http.Transport).CloseIdleConnections()
+               }
+       }
+
        // Make the request
        var resp *http.Response
        if resp, err = c.Client.Do(req); err != nil {