9463: Fixed some integration tests broken because of the replication_desired addition...

[arvados.git] / sdk / go / arvadosclient / arvadosclient.go

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 18e1074bf6f6c801c21593bc57586a902807f8b4..8cdfa484bd96df3f479c7a40c8bb6692eddc1da5 100644 (file)
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -32,6 +32,8 @@ var ErrInvalidArgument = errors.New("Invalid argument")
 // such failures by always using a new or recently active socket.
 var MaxIdleConnectionDuration = 30 * time.Second
 
+var RetryDelay = 2 * time.Second
+
 // Indicates an error that was returned by the API server.
 type APIServerError struct {
        // Address of server returning error, of the form "host:port".
@@ -65,6 +67,9 @@ type Dict map[string]interface{}
 
 // Information about how to contact the Arvados server
 type ArvadosClient struct {
+       // https
+       Scheme string
+
        // Arvados API server, form "host:port"
        ApiServer string
 
@@ -85,6 +90,9 @@ type ArvadosClient struct {
        DiscoveryDoc Dict
 
        lastClosedIdlesAt time.Time
+
+       // Number of retries
+       Retries int
 }
 
 // Create a new ArvadosClient, initialized with standard Arvados environment
@@ -96,12 +104,14 @@ func MakeArvadosClient() (ac ArvadosClient, err error) {
        external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
 
        ac = ArvadosClient{
+               Scheme:      "https",
                ApiServer:   os.Getenv("ARVADOS_API_HOST"),
                ApiToken:    os.Getenv("ARVADOS_API_TOKEN"),
                ApiInsecure: insecure,
                Client: &http.Client{Transport: &http.Transport{
                        TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
-               External: external}
+               External: external,
+               Retries:  2}
 
        if ac.ApiServer == "" {
                return ac, MissingArvadosApiHost
@@ -118,10 +128,12 @@ func MakeArvadosClient() (ac ArvadosClient, err error) {
 // CallRaw is the same as Call() but returns a Reader that reads the
 // response body, instead of taking an output object.
 func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) {
-       var req *http.Request
-
+       scheme := c.Scheme
+       if scheme == "" {
+               scheme = "https"
+       }
        u := url.URL{
-               Scheme: "https",
+               Scheme: scheme,
                Host:   c.ApiServer}
 
        if resourceType != API_DISCOVERY_RESOURCE {
@@ -151,27 +163,15 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
                }
        }
 
-       if method == "GET" || method == "HEAD" {
-               u.RawQuery = vals.Encode()
-               if req, err = http.NewRequest(method, u.String(), nil); err != nil {
-                       return nil, err
-               }
-       } else {
-               if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
-                       return nil, err
-               }
-               req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+       retryable := false
+       switch method {
+       case "GET", "HEAD", "PUT", "OPTIONS", "DELETE":
+               retryable = true
        }
 
-       // Add api token header
-       req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
-       if c.External {
-               req.Header.Add("X-External-Client", "1")
-       }
-
-       // POST and DELETE are not safe to retry automatically, so we minimize
-       // such failures by always using a new or recently active socket
-       if method == "POST" || method == "DELETE" {
+       // Non-retryable methods such as POST are not safe to retry automatically,
+       // so we minimize such failures by always using a new or recently active socket
+       if !retryable {
                if time.Since(c.lastClosedIdlesAt) > MaxIdleConnectionDuration {
                        c.lastClosedIdlesAt = time.Now()
                        c.Client.Transport.(*http.Transport).CloseIdleConnections()
@@ -179,17 +179,57 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
        }
 
        // Make the request
+       var req *http.Request
        var resp *http.Response
-       if resp, err = c.Client.Do(req); err != nil {
-               return nil, err
-       }
 
-       if resp.StatusCode == http.StatusOK {
-               return resp.Body, nil
+       for attempt := 0; attempt <= c.Retries; attempt++ {
+               if method == "GET" || method == "HEAD" {
+                       u.RawQuery = vals.Encode()
+                       if req, err = http.NewRequest(method, u.String(), nil); err != nil {
+                               return nil, err
+                       }
+               } else {
+                       if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
+                               return nil, err
+                       }
+                       req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+               }
+
+               // Add api token header
+               req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
+               if c.External {
+                       req.Header.Add("X-External-Client", "1")
+               }
+
+               resp, err = c.Client.Do(req)
+               if err != nil {
+                       if retryable {
+                               time.Sleep(RetryDelay)
+                               continue
+                       } else {
+                               return nil, err
+                       }
+               }
+
+               if resp.StatusCode == http.StatusOK {
+                       return resp.Body, nil
+               }
+
+               defer resp.Body.Close()
+
+               switch resp.StatusCode {
+               case 408, 409, 422, 423, 500, 502, 503, 504:
+                       time.Sleep(RetryDelay)
+                       continue
+               default:
+                       return nil, newAPIServerError(c.ApiServer, resp)
+               }
        }
 
-       defer resp.Body.Close()
-       return nil, newAPIServerError(c.ApiServer, resp)
+       if resp != nil {
+               return nil, newAPIServerError(c.ApiServer, resp)
+       }
+       return nil, err
 }
 
 func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError {
@@ -233,7 +273,7 @@ func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError
 // Returns a non-nil error if an error occurs making the API call, the
 // API responds with a non-successful HTTP status, or an error occurs
 // parsing the response body.
-func (c ArvadosClient) Call(method string, resourceType string, uuid string, action string, parameters Dict, output interface{}) error {
+func (c ArvadosClient) Call(method, resourceType, uuid, action string, parameters Dict, output interface{}) error {
        reader, err := c.CallRaw(method, resourceType, uuid, action, parameters)
        if reader != nil {
                defer reader.Close()