projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Merge branch '10032-cwl-spinup' refs #10032
[arvados.git]
/
sdk
/
go
/
keepclient
/
keepclient.go
diff --git
a/sdk/go/keepclient/keepclient.go
b/sdk/go/keepclient/keepclient.go
index a018eb40f787d963ce571b50f6f87655882bdf6d..58f3ffb8348ff7b5f9d9588e6455ae7c9e9ff18a 100644
(file)
--- a/
sdk/go/keepclient/keepclient.go
+++ b/
sdk/go/keepclient/keepclient.go
@@
-11,7
+11,6
@@
import (
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
- "log"
"net/http"
"regexp"
"strconv"
"net/http"
"regexp"
"strconv"
@@
-68,7
+67,6
@@
const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
Want_replicas int
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
Want_replicas int
- Using_proxy bool
localRoots *map[string]string
writableLocalRoots *map[string]string
gatewayRoots *map[string]string
localRoots *map[string]string
writableLocalRoots *map[string]string
gatewayRoots *map[string]string
@@
-78,6
+76,9
@@
type KeepClient struct {
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
+
+ // Any non-disk typed services found in the list of keepservers?
+ foundNonDiskSvc bool
}
// MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
}
// MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
@@
-101,7
+102,6
@@
func New(arv *arvadosclient.ArvadosClient) *KeepClient {
kc := &KeepClient{
Arvados: arv,
Want_replicas: defaultReplicationLevel,
kc := &KeepClient{
Arvados: arv,
Want_replicas: defaultReplicationLevel,
- Using_proxy: false,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
Retries: 2,
Client: &http.Client{Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: arv.ApiInsecure}}},
Retries: 2,
@@
-169,10
+169,14
@@
func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
var errs []string
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
var errs []string
- var count404 int
tries_remaining := 1 + kc.Retries
tries_remaining := 1 + kc.Retries
+
serversToTry := kc.getSortedRoots(locator)
serversToTry := kc.getSortedRoots(locator)
+
+ numServers := len(serversToTry)
+ count404 := 0
+
var retryList []string
for tries_remaining > 0 {
var retryList []string
for tries_remaining > 0 {
@@
-196,7
+200,7
@@
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
retryList = append(retryList, host)
} else if resp.StatusCode != http.StatusOK {
var respbody []byte
retryList = append(retryList, host)
} else if resp.StatusCode != http.StatusOK {
var respbody []byte
- respbody, _ = ioutil.ReadAll(&io.LimitedReader{
resp.Body,
4096})
+ respbody, _ = ioutil.ReadAll(&io.LimitedReader{
R: resp.Body, N:
4096})
resp.Body.Close()
errs = append(errs, fmt.Sprintf("%s: HTTP %d %q",
url, resp.StatusCode, bytes.TrimSpace(respbody)))
resp.Body.Close()
errs = append(errs, fmt.Sprintf("%s: HTTP %d %q",
url, resp.StatusCode, bytes.TrimSpace(respbody)))
@@
-228,10
+232,10
@@
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
}
serversToTry = retryList
}
}
serversToTry = retryList
}
-
log.
Printf("DEBUG: %s %s failed: %v", method, locator, errs)
+
Debug
Printf("DEBUG: %s %s failed: %v", method, locator, errs)
var err error
var err error
- if count404 ==
len(kc.getSortedRoots(locator))
{
+ if count404 ==
numServers
{
err = BlockNotFound
} else {
err = &ErrNotFound{multipleResponseError{
err = BlockNotFound
} else {
err = &ErrNotFound{multipleResponseError{
@@
-348,7
+352,7
@@
func (kc *KeepClient) WritableLocalRoots() map[string]string {
// caller can reuse/modify them after SetServiceRoots returns, but
// they should not be modified by any other goroutine while
// SetServiceRoots is running.
// caller can reuse/modify them after SetServiceRoots returns, but
// they should not be modified by any other goroutine while
// SetServiceRoots is running.
-func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals
map[string]string
, newGateways map[string]string) {
+func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals, newGateways map[string]string) {
locals := make(map[string]string)
for uuid, root := range newLocals {
locals[uuid] = root
locals := make(map[string]string)
for uuid, root := range newLocals {
locals[uuid] = root