X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/77b2a1d82d032f7e29f9c477d32127a4435ec13e..80689aac71f6e3e9b103f0f6b668bd173a76554f:/sdk/go/keepclient/keepclient.go

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 2f809b3256..4f84afca61 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -4,14 +4,12 @@ package keepclient
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -68,16 +66,19 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
 type KeepClient struct {
 	Arvados            *arvadosclient.ArvadosClient
 	Want_replicas      int
-	Using_proxy        bool
 	localRoots         *map[string]string
 	writableLocalRoots *map[string]string
 	gatewayRoots       *map[string]string
 	lock               sync.RWMutex
 	Client             *http.Client
 	Retries            int
+	BlockCache         *BlockCache
 
 	// set to 1 if all writable services are of disk type, otherwise 0
 	replicasPerService int
+
+	// Any non-disk typed services found in the list of keepservers?
+	foundNonDiskSvc bool
 }
 
 // MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
@@ -101,9 +102,8 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
 	kc := &KeepClient{
 		Arvados:       arv,
 		Want_replicas: defaultReplicationLevel,
-		Using_proxy:   false,
 		Client: &http.Client{Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
+			TLSClientConfig: arvadosclient.MakeTLSConfig(arv.ApiInsecure)}},
 		Retries: 2,
 	}
 	return kc
@@ -168,6 +168,10 @@ func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error
 }
 
 func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
+	if strings.HasPrefix(locator, "d41d8cd98f00b204e9800998ecf8427e+0") {
+		return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
+	}
+
 	var errs []string
 
 	tries_remaining := 1 + kc.Retries
@@ -200,7 +204,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
 				retryList = append(retryList, host)
 			} else if resp.StatusCode != http.StatusOK {
 				var respbody []byte
-				respbody, _ = ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
+				respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
 				resp.Body.Close()
 				errs = append(errs, fmt.Sprintf("%s: HTTP %d %q",
 					url, resp.StatusCode, bytes.TrimSpace(respbody)))
@@ -232,7 +236,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
 		}
 		serversToTry = retryList
 	}
-	log.Printf("DEBUG: %s %s failed: %v", method, locator, errs)
+	DebugPrintf("DEBUG: %s %s failed: %v", method, locator, errs)
 
 	var err error
 	if count404 == numServers {
@@ -352,7 +356,7 @@ func (kc *KeepClient) WritableLocalRoots() map[string]string {
 // caller can reuse/modify them after SetServiceRoots returns, but
 // they should not be modified by any other goroutine while
 // SetServiceRoots is running.
-func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals map[string]string, newGateways map[string]string) {
+func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals, newGateways map[string]string) {
 	locals := make(map[string]string)
 	for uuid, root := range newLocals {
 		locals[uuid] = root
@@ -403,6 +407,14 @@ func (kc *KeepClient) getSortedRoots(locator string) []string {
 	return found
 }
 
+func (kc *KeepClient) cache() *BlockCache {
+	if kc.BlockCache != nil {
+		return kc.BlockCache
+	} else {
+		return DefaultBlockCache
+	}
+}
+
 type Locator struct {
 	Hash  string
 	Size  int      // -1 if data size is not known