Merge branch '8345-revert-llfuse-to-0.41.1'

[arvados.git] / sdk / go / keepclient / keepclient.go

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index a018eb40f787d963ce571b50f6f87655882bdf6d..baf4bac02444170446c91a61c0b7469813bf308c 100644 (file)
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -4,14 +4,12 @@ package keepclient
 import (
        "bytes"
        "crypto/md5"
-       "crypto/tls"
        "errors"
        "fmt"
        "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
        "git.curoverse.com/arvados.git/sdk/go/streamer"
        "io"
        "io/ioutil"
-       "log"
        "net/http"
        "regexp"
        "strconv"
@@ -68,7 +66,6 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
 type KeepClient struct {
        Arvados            *arvadosclient.ArvadosClient
        Want_replicas      int
-       Using_proxy        bool
        localRoots         *map[string]string
        writableLocalRoots *map[string]string
        gatewayRoots       *map[string]string
@@ -78,6 +75,9 @@ type KeepClient struct {
 
        // set to 1 if all writable services are of disk type, otherwise 0
        replicasPerService int
+
+       // Any non-disk typed services found in the list of keepservers?
+       foundNonDiskSvc bool
 }
 
 // MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
@@ -101,9 +101,8 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
        kc := &KeepClient{
                Arvados:       arv,
                Want_replicas: defaultReplicationLevel,
-               Using_proxy:   false,
                Client: &http.Client{Transport: &http.Transport{
-                       TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
+                       TLSClientConfig: arvadosclient.MakeTLSConfig(arv.ApiInsecure)}},
                Retries: 2,
        }
        return kc
@@ -168,11 +167,19 @@ func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error
 }
 
 func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
+       if strings.HasPrefix(locator, "d41d8cd98f00b204e9800998ecf8427e+0") {
+               return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
+       }
+
        var errs []string
-       var count404 int
 
        tries_remaining := 1 + kc.Retries
+
        serversToTry := kc.getSortedRoots(locator)
+
+       numServers := len(serversToTry)
+       count404 := 0
+
        var retryList []string
 
        for tries_remaining > 0 {
@@ -196,7 +203,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
                                retryList = append(retryList, host)
                        } else if resp.StatusCode != http.StatusOK {
                                var respbody []byte
-                               respbody, _ = ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
+                               respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
                                resp.Body.Close()
                                errs = append(errs, fmt.Sprintf("%s: HTTP %d %q",
                                        url, resp.StatusCode, bytes.TrimSpace(respbody)))
@@ -228,10 +235,10 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
                }
                serversToTry = retryList
        }
-       log.Printf("DEBUG: %s %s failed: %v", method, locator, errs)
+       DebugPrintf("DEBUG: %s %s failed: %v", method, locator, errs)
 
        var err error
-       if count404 == len(kc.getSortedRoots(locator)) {
+       if count404 == numServers {
                err = BlockNotFound
        } else {
                err = &ErrNotFound{multipleResponseError{
@@ -348,7 +355,7 @@ func (kc *KeepClient) WritableLocalRoots() map[string]string {
 // caller can reuse/modify them after SetServiceRoots returns, but
 // they should not be modified by any other goroutine while
 // SetServiceRoots is running.
-func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals map[string]string, newGateways map[string]string) {
+func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals, newGateways map[string]string) {
        locals := make(map[string]string)
        for uuid, root := range newLocals {
                locals[uuid] = root