Arvados-DCO-1.1-Signed-off-by: Radhika Chippada <radhika@curoverse.com>

[arvados.git] / sdk / go / keepclient / support.go

diff --git a/sdk/go/keepclient/support.go b/sdk/go/keepclient/support.go
index 22447794f1c7f93c4eca6fcdec83d254cbc1740b..49ef543d872f94d169c2e76b422cffee76ef86ed 100644 (file)
--- a/sdk/go/keepclient/support.go
+++ b/sdk/go/keepclient/support.go
@@ -1,17 +1,23 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
 package keepclient
 
 import (
        "crypto/md5"
        "errors"
        "fmt"
-       "git.curoverse.com/arvados.git/sdk/go/streamer"
        "io"
        "io/ioutil"
+       "log"
        "math/rand"
-       "net"
        "net/http"
+       "os"
        "strings"
-       "time"
+
+       "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+       "git.curoverse.com/arvados.git/sdk/go/streamer"
 )
 
 // Function used to emit debug messages. The easiest way to enable
@@ -19,6 +25,12 @@ import (
 // log.Printf to DebugPrintf.
 var DebugPrintf = func(string, ...interface{}) {}
 
+func init() {
+       if arvadosclient.StringBool(os.Getenv("ARVADOS_DEBUG")) {
+               DebugPrintf = log.Printf
+       }
+}
+
 type keepService struct {
        Uuid     string `json:"uuid"`
        Hostname string `json:"service_host"`
@@ -33,54 +45,6 @@ func Md5String(s string) string {
        return fmt.Sprintf("%x", md5.Sum([]byte(s)))
 }
 
-// Set timeouts applicable when connecting to non-disk services
-// (assumed to be over the Internet).
-func (this *KeepClient) setClientSettingsNonDisk() {
-       if this.Client.Timeout == 0 {
-               // Maximum time to wait for a complete response
-               this.Client.Timeout = 300 * time.Second
-
-               // TCP and TLS connection settings
-               this.Client.Transport = &http.Transport{
-                       Dial: (&net.Dialer{
-                               // The maximum time to wait to set up
-                               // the initial TCP connection.
-                               Timeout: 30 * time.Second,
-
-                               // The TCP keep alive heartbeat
-                               // interval.
-                               KeepAlive: 120 * time.Second,
-                       }).Dial,
-
-                       TLSHandshakeTimeout: 10 * time.Second,
-               }
-       }
-}
-
-// Set timeouts applicable when connecting to keepstore services directly
-// (assumed to be on the local network).
-func (this *KeepClient) setClientSettingsDisk() {
-       if this.Client.Timeout == 0 {
-               // Maximum time to wait for a complete response
-               this.Client.Timeout = 20 * time.Second
-
-               // TCP and TLS connection timeouts
-               this.Client.Transport = &http.Transport{
-                       Dial: (&net.Dialer{
-                               // The maximum time to wait to set up
-                               // the initial TCP connection.
-                               Timeout: 2 * time.Second,
-
-                               // The TCP keep alive heartbeat
-                               // interval.
-                               KeepAlive: 180 * time.Second,
-                       }).Dial,
-
-                       TLSHandshakeTimeout: 4 * time.Second,
-               }
-       }
-}
-
 type svcList struct {
        Items []keepService `json:"items"`
 }
@@ -108,8 +72,8 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
 
        req.ContentLength = expectedLength
        if expectedLength > 0 {
-               // http.Client.Do will close the body ReadCloser when it is
-               // done with it.
+               // Do() will close the body ReadCloser when it is done
+               // with it.
                req.Body = body
        } else {
                // "For client requests, a value of 0 means unknown if Body is
@@ -124,7 +88,7 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
        req.Header.Add(X_Keep_Desired_Replicas, fmt.Sprint(this.Want_replicas))
 
        var resp *http.Response
-       if resp, err = this.Client.Do(req); err != nil {
+       if resp, err = this.httpClient().Do(req); err != nil {
                DebugPrintf("DEBUG: [%08x] Upload failed %v error: %v", requestID, url, err.Error())
                upload_status <- uploadStatus{err, url, 0, 0, ""}
                return
@@ -147,6 +111,9 @@ func (this *KeepClient) uploadToKeepServer(host string, hash string, body io.Rea
                DebugPrintf("DEBUG: [%08x] Upload %v success", requestID, url)
                upload_status <- uploadStatus{nil, url, resp.StatusCode, rep, response}
        } else {
+               if resp.StatusCode >= 300 && response == "" {
+                       response = resp.Status
+               }
                DebugPrintf("DEBUG: [%08x] Upload %v error: %v response: %v", requestID, url, resp.StatusCode, response)
                upload_status <- uploadStatus{errors.New(resp.Status), url, resp.StatusCode, rep, response}
        }
@@ -196,6 +163,8 @@ func (this *KeepClient) putReplicas(
        retriesRemaining := 1 + this.Retries
        var retryServers []string
 
+       lastError := make(map[string]string)
+
        for retriesRemaining > 0 {
                retriesRemaining -= 1
                next_server = 0
@@ -210,7 +179,12 @@ func (this *KeepClient) putReplicas(
                                        active += 1
                                } else {
                                        if active == 0 && retriesRemaining == 0 {
-                                               return locator, replicasDone, InsufficientReplicasError
+                                               msg := "Could not write sufficient replicas: "
+                                               for _, resp := range lastError {
+                                                       msg += resp + "; "
+                                               }
+                                               msg = msg[:len(msg)-2]
+                                               return locator, replicasDone, InsufficientReplicasError(errors.New(msg))
                                        } else {
                                                break
                                        }
@@ -229,7 +203,16 @@ func (this *KeepClient) putReplicas(
                                        replicasDone += status.replicas_stored
                                        replicasTodo -= status.replicas_stored
                                        locator = status.response
-                               } else if status.statusCode == 0 || status.statusCode == 408 || status.statusCode == 429 ||
+                                       delete(lastError, status.url)
+                               } else {
+                                       msg := fmt.Sprintf("[%d] %s", status.statusCode, status.response)
+                                       if len(msg) > 100 {
+                                               msg = msg[:100]
+                                       }
+                                       lastError[status.url] = msg
+                               }
+
+                               if status.statusCode == 0 || status.statusCode == 408 || status.statusCode == 429 ||
                                        (status.statusCode >= 500 && status.statusCode != 503) {
                                        // Timeout, too many requests, or other server side failure
                                        // Do not retry when status code is 503, which means the keep server is full