X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/0c888bcc93b559339c8abbce784bdcc44746bca2..fa78e264ef585f02348f4f5c0a7183746a708a8f:/sdk/go/keepclient/keepclient.go diff --git a/sdk/go/keepclient/keepclient.go b/sdk/go/keepclient/keepclient.go index 54a4a374b9..8eff377ce6 100644 --- a/sdk/go/keepclient/keepclient.go +++ b/sdk/go/keepclient/keepclient.go @@ -20,8 +20,9 @@ import ( "sync" "time" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/asyncbuf" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/asyncbuf" + "git.arvados.org/arvados.git/sdk/go/httpserver" ) // A Keep "block" is 64MB. @@ -99,6 +100,8 @@ type KeepClient struct { HTTPClient HTTPClient Retries int BlockCache *BlockCache + RequestID string + StorageClasses []string // set to 1 if all writable services are of disk type, otherwise 0 replicasPerService int @@ -195,9 +198,20 @@ func (kc *KeepClient) PutR(r io.Reader) (locator string, replicas int, err error } } -func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, int64, string, error) { +func (kc *KeepClient) getOrHead(method string, locator string, header http.Header) (io.ReadCloser, int64, string, http.Header, error) { if strings.HasPrefix(locator, "d41d8cd98f00b204e9800998ecf8427e+0") { - return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil + return ioutil.NopCloser(bytes.NewReader(nil)), 0, "", nil, nil + } + + reqid := kc.getRequestID() + + var expectLength int64 + if parts := strings.SplitN(locator, "+", 3); len(parts) < 2 { + expectLength = -1 + } else if n, err := strconv.ParseInt(parts[1], 10, 64); err != nil { + expectLength = -1 + } else { + expectLength = n } var errs []string @@ -223,14 +237,24 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i errs = append(errs, fmt.Sprintf("%s: %v", url, err)) continue } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken)) + for k, v := range header { + req.Header[k] = append([]string(nil), v...) + } + if req.Header.Get("Authorization") == "" { + req.Header.Set("Authorization", "OAuth2 "+kc.Arvados.ApiToken) + } + if req.Header.Get("X-Request-Id") == "" { + req.Header.Set("X-Request-Id", reqid) + } resp, err := kc.httpClient().Do(req) if err != nil { // Probably a network error, may be transient, // can try again. errs = append(errs, fmt.Sprintf("%s: %v", url, err)) retryList = append(retryList, host) - } else if resp.StatusCode != http.StatusOK { + continue + } + if resp.StatusCode != http.StatusOK { var respbody []byte respbody, _ = ioutil.ReadAll(&io.LimitedReader{R: resp.Body, N: 4096}) resp.Body.Close() @@ -247,24 +271,28 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i } else if resp.StatusCode == 404 { count404++ } - } else if resp.ContentLength < 0 { - // Missing Content-Length - resp.Body.Close() - return nil, 0, "", fmt.Errorf("Missing Content-Length of block") - } else { - // Success. - if method == "GET" { - return HashCheckingReader{ - Reader: resp.Body, - Hash: md5.New(), - Check: locator[0:32], - }, resp.ContentLength, url, nil - } else { + continue + } + if expectLength < 0 { + if resp.ContentLength < 0 { resp.Body.Close() - return nil, resp.ContentLength, url, nil + return nil, 0, "", nil, fmt.Errorf("error reading %q: no size hint, no Content-Length header in response", locator) } + expectLength = resp.ContentLength + } else if resp.ContentLength >= 0 && expectLength != resp.ContentLength { + resp.Body.Close() + return nil, 0, "", nil, fmt.Errorf("error reading %q: size hint %d != Content-Length %d", locator, expectLength, resp.ContentLength) } - + // Success + if method == "GET" { + return HashCheckingReader{ + Reader: resp.Body, + Hash: md5.New(), + Check: locator[0:32], + }, expectLength, url, resp.Header, nil + } + resp.Body.Close() + return nil, expectLength, url, resp.Header, nil } serversToTry = retryList } @@ -279,7 +307,29 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i isTemp: len(serversToTry) > 0, }} } - return nil, 0, "", err + return nil, 0, "", nil, err +} + +// LocalLocator returns a locator equivalent to the one supplied, but +// with a valid signature from the local cluster. If the given locator +// already has a local signature, it is returned unchanged. +func (kc *KeepClient) LocalLocator(locator string) (string, error) { + if !strings.Contains(locator, "+R") { + // Either it has +A, or it's unsigned and we assume + // it's a local locator on a site with signatures + // disabled. + return locator, nil + } + sighdr := fmt.Sprintf("local, time=%s", time.Now().UTC().Format(time.RFC3339)) + _, _, url, hdr, err := kc.getOrHead("HEAD", locator, http.Header{"X-Keep-Signature": []string{sighdr}}) + if err != nil { + return "", err + } + loc := hdr.Get("X-Keep-Locator") + if loc == "" { + return "", fmt.Errorf("missing X-Keep-Locator header in HEAD response from %s", url) + } + return loc, nil } // Get() retrieves a block, given a locator. Returns a reader, the @@ -290,7 +340,8 @@ func (kc *KeepClient) getOrHead(method string, locator string) (io.ReadCloser, i // reader returned by this method will return a BadChecksum error // instead of EOF. func (kc *KeepClient) Get(locator string) (io.ReadCloser, int64, string, error) { - return kc.getOrHead("GET", locator) + rdr, size, url, _, err := kc.getOrHead("GET", locator, nil) + return rdr, size, url, err } // ReadAt() retrieves a portion of block from the cache if it's @@ -307,7 +358,7 @@ func (kc *KeepClient) ReadAt(locator string, p []byte, off int) (int, error) { // Returns the data size (content length) reported by the Keep service // and the URI reporting the data size. func (kc *KeepClient) Ask(locator string) (int64, string, error) { - _, size, url, err := kc.getOrHead("HEAD", locator) + _, size, url, _, err := kc.getOrHead("HEAD", locator, nil) return size, url, err } @@ -334,7 +385,8 @@ func (kc *KeepClient) GetIndex(keepServiceUUID, prefix string) (io.Reader, error return nil, err } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", kc.Arvados.ApiToken)) + req.Header.Add("Authorization", "OAuth2 "+kc.Arvados.ApiToken) + req.Header.Set("X-Request-Id", kc.getRequestID()) resp, err := kc.httpClient().Do(req) if err != nil { return nil, err @@ -440,9 +492,8 @@ func (kc *KeepClient) getSortedRoots(locator string) []string { func (kc *KeepClient) cache() *BlockCache { if kc.BlockCache != nil { return kc.BlockCache - } else { - return DefaultBlockCache } + return DefaultBlockCache } func (kc *KeepClient) ClearBlockCache() { @@ -493,36 +544,40 @@ func (kc *KeepClient) httpClient() HTTPClient { keepAlive = DefaultKeepAlive } - transport, ok := http.DefaultTransport.(*http.Transport) - if ok { - copy := *transport - transport = © - } else { - // Evidently the application has replaced - // http.DefaultTransport with a different type, so we - // need to build our own from scratch using the Go 1.8 - // defaults. - transport = &http.Transport{ + c := &http.Client{ + Timeout: requestTimeout, + // It's not safe to copy *http.DefaultTransport + // because it has a mutex (which might be locked) + // protecting a private map (which might not be nil). + // So we build our own, using the Go 1.12 default + // values, ignoring any changes the application has + // made to http.DefaultTransport. + Transport: &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: connectTimeout, + KeepAlive: keepAlive, + DualStack: true, + }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, - ExpectContinueTimeout: time.Second, - } - } - transport.DialContext = (&net.Dialer{ - Timeout: connectTimeout, - KeepAlive: keepAlive, - DualStack: true, - }).DialContext - transport.TLSHandshakeTimeout = tlsTimeout - transport.TLSClientConfig = arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure) - c := &http.Client{ - Timeout: requestTimeout, - Transport: transport, + TLSHandshakeTimeout: tlsTimeout, + ExpectContinueTimeout: 1 * time.Second, + TLSClientConfig: arvadosclient.MakeTLSConfig(kc.Arvados.ApiInsecure), + }, } defaultClient[kc.Arvados.ApiInsecure][kc.foundNonDiskSvc] = c return c } +var reqIDGen = httpserver.IDGenerator{Prefix: "req-"} + +func (kc *KeepClient) getRequestID() string { + if kc.RequestID != "" { + return kc.RequestID + } + return reqIDGen.Next() +} + type Locator struct { Hash string Size int // -1 if data size is not known