5538: retry failed arvados api requests when appropriate.

[arvados.git] / sdk / go / arvadosclient / arvadosclient.go

diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index af8bce4d4a7336cac0c66fb07292aaa203b4b0e6..efe1583297a50824fe2e04930d6a88b8bf63b6f6 100644 (file)
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -58,6 +58,9 @@ type Dict map[string]interface{}
 
 // Information about how to contact the Arvados server
 type ArvadosClient struct {
+       // https
+       Scheme string
+
        // Arvados API server, form "host:port"
        ApiServer string
 
@@ -76,44 +79,28 @@ type ArvadosClient struct {
 
        // Discovery document
        DiscoveryDoc Dict
-}
 
-// APIConfig struct consists of:
-//    APIToken        string
-//    APIHost         string
-//    APIHostInsecure bool
-//    ExternalClient  bool
-type APIConfig struct {
-       APIToken        string
-       APIHost         string
-       APIHostInsecure bool
-       ExternalClient  bool
+       // Number of retries
+       Retries int
 }
 
-// Create a new ArvadosClient, initialized with standard Arvados environment variables
-// ARVADOS_API_HOST, ARVADOS_API_TOKEN, ARVADOS_API_HOST_INSECURE, ARVADOS_EXTERNAL_CLIENT.
+// Create a new ArvadosClient, initialized with standard Arvados environment
+// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
+// ARVADOS_API_HOST_INSECURE.
 func MakeArvadosClient() (ac ArvadosClient, err error) {
-       var config APIConfig
-       config.APIToken = os.Getenv("ARVADOS_API_TOKEN")
-       config.APIHost = os.Getenv("ARVADOS_API_HOST")
-
        var matchTrue = regexp.MustCompile("^(?i:1|yes|true)$")
+       insecure := matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
+       external := matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
 
-       config.APIHostInsecure = matchTrue.MatchString(os.Getenv("ARVADOS_API_HOST_INSECURE"))
-       config.ExternalClient = matchTrue.MatchString(os.Getenv("ARVADOS_EXTERNAL_CLIENT"))
-
-       return New(config)
-}
-
-// Create a new ArvadosClient, using the given input parameters.
-func New(config APIConfig) (ac ArvadosClient, err error) {
        ac = ArvadosClient{
-               ApiServer:   config.APIHost,
-               ApiToken:    config.APIToken,
-               ApiInsecure: config.APIHostInsecure,
+               Scheme:      "https",
+               ApiServer:   os.Getenv("ARVADOS_API_HOST"),
+               ApiToken:    os.Getenv("ARVADOS_API_TOKEN"),
+               ApiInsecure: insecure,
                Client: &http.Client{Transport: &http.Transport{
-                       TLSClientConfig: &tls.Config{InsecureSkipVerify: config.APIHostInsecure}}},
-               External: config.ExternalClient}
+                       TLSClientConfig: &tls.Config{InsecureSkipVerify: insecure}}},
+               External: external,
+               Retries:  2}
 
        if ac.ApiServer == "" {
                return ac, MissingArvadosApiHost
@@ -128,10 +115,12 @@ func New(config APIConfig) (ac ArvadosClient, err error) {
 // CallRaw is the same as Call() but returns a Reader that reads the
 // response body, instead of taking an output object.
 func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string, action string, parameters Dict) (reader io.ReadCloser, err error) {
-       var req *http.Request
-
+       scheme := c.Scheme
+       if scheme == "" {
+               scheme = "https"
+       }
        u := url.URL{
-               Scheme: "https",
+               Scheme: scheme,
                Host:   c.ApiServer}
 
        if resourceType != API_DISCOVERY_RESOURCE {
@@ -161,36 +150,71 @@ func (c ArvadosClient) CallRaw(method string, resourceType string, uuid string,
                }
        }
 
-       if method == "GET" || method == "HEAD" {
-               u.RawQuery = vals.Encode()
-               if req, err = http.NewRequest(method, u.String(), nil); err != nil {
-                       return nil, err
+       // Make the request
+       remainingTries := 1 + c.Retries
+       var req *http.Request
+       var resp *http.Response
+       var errs []string
+       var badResp bool
+
+       for remainingTries > 0 {
+               if method == "GET" || method == "HEAD" {
+                       u.RawQuery = vals.Encode()
+                       if req, err = http.NewRequest(method, u.String(), nil); err != nil {
+                               return nil, err
+                       }
+               } else {
+                       if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
+                               return nil, err
+                       }
+                       req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
                }
-       } else {
-               if req, err = http.NewRequest(method, u.String(), bytes.NewBufferString(vals.Encode())); err != nil {
-                       return nil, err
+
+               // Add api token header
+               req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
+               if c.External {
+                       req.Header.Add("X-External-Client", "1")
                }
-               req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
-       }
 
-       // Add api token header
-       req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", c.ApiToken))
-       if c.External {
-               req.Header.Add("X-External-Client", "1")
-       }
+               resp, err = c.Client.Do(req)
+               if err != nil {
+                       if method == "GET" || method == "HEAD" || method == "PUT" {
+                               errs = append(errs, err.Error())
+                               badResp = false
+                               remainingTries -= 1
+                               continue
+                       } else {
+                               return nil, err
+                       }
+               }
 
-       // Make the request
-       var resp *http.Response
-       if resp, err = c.Client.Do(req); err != nil {
-               return nil, err
-       }
+               if resp.StatusCode == http.StatusOK {
+                       return resp.Body, nil
+               }
 
-       if resp.StatusCode == http.StatusOK {
-               return resp.Body, nil
+               defer resp.Body.Close()
+
+               if resp.StatusCode == 408 ||
+                       resp.StatusCode == 409 ||
+                       resp.StatusCode == 422 ||
+                       resp.StatusCode == 423 ||
+                       resp.StatusCode == 500 ||
+                       resp.StatusCode == 502 ||
+                       resp.StatusCode == 503 ||
+                       resp.StatusCode == 504 {
+                       badResp = true
+                       remainingTries -= 1
+                       continue
+               } else {
+                       return nil, newAPIServerError(c.ApiServer, resp)
+               }
        }
 
-       defer resp.Body.Close()
-       return nil, newAPIServerError(c.ApiServer, resp)
+       if badResp {
+               return nil, newAPIServerError(c.ApiServer, resp)
+       } else {
+               return nil, fmt.Errorf("%v", errs)
+       }
 }
 
 func newAPIServerError(ServerAddress string, resp *http.Response) APIServerError {