X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/ce30948199736d45112ee9103642c22f59f84997..965565ddc62635928a6b043158fd683738961c8c:/sdk/go/keepclient/keepclient.go

diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go
index 67c304deaf..79a87156a6 100644
--- a/sdk/go/keepclient/keepclient.go
+++ b/sdk/go/keepclient/keepclient.go
@@ -4,14 +4,12 @@ package keepclient
 import (
 	"bytes"
 	"crypto/md5"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
 	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	"io"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"regexp"
 	"strconv"
@@ -22,7 +20,33 @@ import (
 // A Keep "block" is 64MB.
 const BLOCKSIZE = 64 * 1024 * 1024
 
-var BlockNotFound = errors.New("Block not found")
+// Error interface with an error and boolean indicating whether the error is temporary
+type Error interface {
+	error
+	Temporary() bool
+}
+
+// multipleResponseError is of type Error
+type multipleResponseError struct {
+	error
+	isTemp bool
+}
+
+func (e *multipleResponseError) Temporary() bool {
+	return e.isTemp
+}
+
+// BlockNotFound is a multipleResponseError where isTemp is false
+var BlockNotFound = &ErrNotFound{multipleResponseError{
+	error:  errors.New("Block not found"),
+	isTemp: false,
+}}
+
+// ErrNotFound is a multipleResponseError where isTemp can be true or false
+type ErrNotFound struct {
+	multipleResponseError
+}
+
 var InsufficientReplicasError = errors.New("Could not write sufficient replicas")
 var OversizeBlockError = errors.New("Exceeded maximum block size (" + strconv.Itoa(BLOCKSIZE) + ")")
 var MissingArvadosApiHost = errors.New("Missing required environment variable ARVADOS_API_HOST")
@@ -42,7 +66,6 @@ const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
 type KeepClient struct {
 	Arvados            *arvadosclient.ArvadosClient
 	Want_replicas      int
-	Using_proxy        bool
 	localRoots         *map[string]string
 	writableLocalRoots *map[string]string
 	gatewayRoots       *map[string]string
@@ -52,6 +75,9 @@ type KeepClient struct {
 
 	// set to 1 if all writable services are of disk type, otherwise 0
 	replicasPerService int
+
+	// Any non-disk typed services found in the list of keepservers?
+	foundNonDiskSvc bool
 }
 
 // MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
@@ -75,9 +101,8 @@ func New(arv *arvadosclient.ArvadosClient) *KeepClient {
 	kc := &KeepClient{
 		Arvados:       arv,
 		Want_replicas: defaultReplicationLevel,
-		Using_proxy:   false,
 		Client: &http.Client{Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
+			TLSClientConfig: arvadosclient.MakeTLSConfig(arv.ApiInsecure)}},
 		Retries: 2,
 	}
 	return kc
@@ -145,7 +170,12 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
 	var errs []string
 
 	tries_remaining := 1 + kc.Retries
+
 	serversToTry := kc.getSortedRoots(locator)
+
+	numServers := len(serversToTry)
+	count404 := 0
+
 	var retryList []string
 
 	for tries_remaining > 0 {
@@ -169,7 +199,7 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
 				retryList = append(retryList, host)
 			} else if resp.StatusCode != http.StatusOK {
 				var respbody []byte
-				respbody, _ = ioutil.ReadAll(&io.LimitedReader{resp.Body, 4096})
+				respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096})
 				resp.Body.Close()
 				errs = append(errs, fmt.Sprintf("%s: HTTP %d %q",
 					url, resp.StatusCode, bytes.TrimSpace(respbody)))
@@ -181,6 +211,8 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
 					// server side failure, transient
 					// error, can try again.
 					retryList = append(retryList, host)
+				} else if resp.StatusCode == 404 {
+					count404++
 				}
 			} else {
 				// Success.
@@ -199,9 +231,18 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
 		}
 		serversToTry = retryList
 	}
-	log.Printf("DEBUG: %s %s failed: %v", method, locator, errs)
+	DebugPrintf("DEBUG: %s %s failed: %v", method, locator, errs)
 
-	return nil, 0, "", BlockNotFound
+	var err error
+	if count404 == numServers {
+		err = BlockNotFound
+	} else {
+		err = &ErrNotFound{multipleResponseError{
+			error:  fmt.Errorf("%s %s failed: %v", method, locator, errs),
+			isTemp: len(serversToTry) > 0,
+		}}
+	}
+	return nil, 0, "", err
 }
 
 // Get() retrieves a block, given a locator. Returns a reader, the
@@ -310,7 +351,7 @@ func (kc *KeepClient) WritableLocalRoots() map[string]string {
 // caller can reuse/modify them after SetServiceRoots returns, but
 // they should not be modified by any other goroutine while
 // SetServiceRoots is running.
-func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals map[string]string, newGateways map[string]string) {
+func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals, newGateways map[string]string) {
 	locals := make(map[string]string)
 	for uuid, root := range newLocals {
 		locals[uuid] = root