X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/853cb66fe7d3c24e9936b4a49fdd775abc0ed0f4..303424c3f3ed01f65c9f880ff1ff32bc7a46621b:/sdk/go/keepclient/keepclient.go

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 58f3ffb834..b56cc7f724 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -4,11 +4,8 @@ package keepclient
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/tls"
 	"errors"
 	"fmt"
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -16,6 +13,9 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/streamer"
 )
 
 // A Keep "block" is 64MB.
@@ -48,8 +48,11 @@ type ErrNotFound struct {
 	multipleResponseError
 }
 
-var InsufficientReplicasError = errors.New("Could not write sufficient replicas")
-var OversizeBlockError = errors.New("Exceeded maximum block size (" + strconv.Itoa(BLOCKSIZE) + ")")
+type InsufficientReplicasError error
+
+type OversizeBlockError error
+
+var ErrOversizeBlock = OversizeBlockError(errors.New("Exceeded maximum block size (" + strconv.Itoa(BLOCKSIZE) + ")"))
 var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
 var MissingArvadosApiToken = errors.New("Missing required environment variable ARVADOS_API_TOKEN")
 var InvalidLocatorError = errors.New("Invalid locator")
@@ -63,6 +66,10 @@ var ErrIncompleteIndex = errors.New("Got incomplete index")
 const X_Keep_Desired_Replicas = "X-Keep-Desired-Replicas"
 const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
 
+type HTTPClient interface {
+	Do(*http.Request) (*http.Response, error)
+}
+
 // Information about Arvados and Keep servers.
 type KeepClient struct {
 	Arvados            *arvadosclient.ArvadosClient
@@ -71,8 +78,9 @@ type KeepClient struct {
 	writableLocalRoots *map[string]string
 	gatewayRoots       *map[string]string
 	lock               sync.RWMutex
-	Client             *http.Client
+	Client             HTTPClient
 	Retries            int
+	BlockCache         *BlockCache
 
 	// set to 1 if all writable services are of disk type, otherwise 0
 	replicasPerService int
@@ -103,7 +111,7 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
 		Arvados:       arv,
 		Want_replicas: defaultReplicationLevel,
 		Client: &http.Client{Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
+			TLSClientConfig: arvadosclient.MakeTLSConfig(arv.ApiInsecure)}},
 		Retries: 2,
 	}
 	return kc
@@ -115,14 +123,14 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
 // Returns the locator for the written block, the number of replicas
 // written, and an error.
 //
-// Returns an InsufficientReplicas error if 0 <= replicas <
+// Returns an InsufficientReplicasError if 0 <= replicas <
 // kc.Wants_replicas.
 func (kc *KeepClient) PutHR(hash string, r io.Reader, dataBytes int64) (string, int, error) {
 	// Buffer for reads from 'r'
 	var bufsize int
 	if dataBytes > 0 {
 		if dataBytes > BLOCKSIZE {
-			return "", 0, OversizeBlockError
+			return "", 0, ErrOversizeBlock
 		}
 		bufsize = int(dataBytes)
 	} else {
@@ -168,6 +176,10 @@ func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error
 }
 
 func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
+	if strings.HasPrefix(locator, "d41d8cd98f00b204e9800998ecf8427e+0") {
+		return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
+	}
+
 	var errs []string
 
 	tries_remaining := 1 + kc.Retries
@@ -403,6 +415,14 @@ func (kc *KeepClient) getSortedRoots(locator string) []string {
 	return found
 }
 
+func (kc *KeepClient) cache() *BlockCache {
+	if kc.BlockCache != nil {
+		return kc.BlockCache
+	} else {
+		return DefaultBlockCache
+	}
+}
+
 type Locator struct {
 	Hash  string
 	Size  int      // -1 if data size is not known