projects
/
arvados.git
/ blobdiff
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
7709: Merge branch 'master' into 7709-api-rails4
[arvados.git]
/
sdk
/
go
/
keepclient
/
keepclient.go
diff --git
a/sdk/go/keepclient/keepclient.go
b/sdk/go/keepclient/keepclient.go
index 0e6fadcc3548c99d2e85c30219a74ffdce42bf72..4f84afca61a413796fbb222eb5108473c449cfaf 100644
(file)
--- a/
sdk/go/keepclient/keepclient.go
+++ b/
sdk/go/keepclient/keepclient.go
@@
-4,14
+4,12
@@
package keepclient
import (
"bytes"
"crypto/md5"
import (
"bytes"
"crypto/md5"
- "crypto/tls"
"errors"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
"errors"
"fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/streamer"
"io"
"io/ioutil"
- "log"
"net/http"
"regexp"
"strconv"
"net/http"
"regexp"
"strconv"
@@
-68,16
+66,19
@@
const X_Keep_Replicas_Stored = "X-Keep-Replicas-Stored"
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
Want_replicas int
type KeepClient struct {
Arvados *arvadosclient.ArvadosClient
Want_replicas int
- Using_proxy bool
localRoots *map[string]string
writableLocalRoots *map[string]string
gatewayRoots *map[string]string
lock sync.RWMutex
Client *http.Client
Retries int
localRoots *map[string]string
writableLocalRoots *map[string]string
gatewayRoots *map[string]string
lock sync.RWMutex
Client *http.Client
Retries int
+ BlockCache *BlockCache
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
// set to 1 if all writable services are of disk type, otherwise 0
replicasPerService int
+
+ // Any non-disk typed services found in the list of keepservers?
+ foundNonDiskSvc bool
}
// MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
}
// MakeKeepClient creates a new KeepClient by contacting the API server to discover Keep servers.
@@
-101,9
+102,8
@@
func New(arv *arvadosclient.ArvadosClient) *KeepClient {
kc := &KeepClient{
Arvados: arv,
Want_replicas: defaultReplicationLevel,
kc := &KeepClient{
Arvados: arv,
Want_replicas: defaultReplicationLevel,
- Using_proxy: false,
Client: &http.Client{Transport: &http.Transport{
Client: &http.Client{Transport: &http.Transport{
- TLSClientConfig:
&tls.Config{InsecureSkipVerify: arv.ApiInsecure}
}},
+ TLSClientConfig:
arvadosclient.MakeTLSConfig(arv.ApiInsecure)
}},
Retries: 2,
}
return kc
Retries: 2,
}
return kc
@@
-168,6
+168,10
@@
func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error
}
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
}
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) {
+ if strings.HasPrefix(locator, "d41d8cd98f00b204e9800998ecf8427e+0") {
+ return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil
+ }
+
var errs []string
tries_remaining := 1 + kc.Retries
var errs []string
tries_remaining := 1 + kc.Retries
@@
-232,7
+236,7
@@
func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i
}
serversToTry = retryList
}
}
serversToTry = retryList
}
-
log.
Printf("DEBUG: %s %s failed: %v", method, locator, errs)
+
Debug
Printf("DEBUG: %s %s failed: %v", method, locator, errs)
var err error
if count404 == numServers {
var err error
if count404 == numServers {
@@
-352,7
+356,7
@@
func (kc *KeepClient) WritableLocalRoots() map[string]string {
// caller can reuse/modify them after SetServiceRoots returns, but
// they should not be modified by any other goroutine while
// SetServiceRoots is running.
// caller can reuse/modify them after SetServiceRoots returns, but
// they should not be modified by any other goroutine while
// SetServiceRoots is running.
-func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals
map[string]string
, newGateways map[string]string) {
+func (kc *KeepClient) SetServiceRoots(newLocals, newWritableLocals, newGateways map[string]string) {
locals := make(map[string]string)
for uuid, root := range newLocals {
locals[uuid] = root
locals := make(map[string]string)
for uuid, root := range newLocals {
locals[uuid] = root
@@
-403,6
+407,14
@@
func (kc *KeepClient) getSortedRoots(locator string) []string {
return found
}
return found
}
+func (kc *KeepClient) cache() *BlockCache {
+ if kc.BlockCache != nil {
+ return kc.BlockCache
+ } else {
+ return DefaultBlockCache
+ }
+}
+
type Locator struct {
Hash string
Size int // -1 if data size is not known
type Locator struct {
Hash string
Size int // -1 if data size is not known