+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package keepclient
import (
+ "bytes"
"crypto/md5"
"errors"
"fmt"
"testing"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
- "git.curoverse.com/arvados.git/sdk/go/streamer"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
. "gopkg.in/check.v1"
)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 1)
}
type StubPutHandler struct {
- c *C
- expectPath string
- expectApiToken string
- expectBody string
- handled chan string
+ c *C
+ expectPath string
+ expectApiToken string
+ expectBody string
+ expectStorageClass string
+ handled chan string
}
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
var err error
- ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
+ // If we don't explicitly bind it to localhost, ks.listener.Addr() will
+ // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
+ ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
if err != nil {
panic(fmt.Sprintf("Could not listen on any port"))
}
"acbd18db4cc2f85cedef654fccc4a4d8",
"abc123",
"foo",
+ "hot",
make(chan string)}
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
-
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
+ kc.StorageClasses = []string{"hot"}
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
"acbd18db4cc2f85cedef654fccc4a4d8",
"abc123",
"foo",
+ "",
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
-
- tr := streamer.AsyncStreamFromReader(512, reader)
- defer tr.Close()
-
- br1 := tr.MakeStreamReader()
-
- go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
-
- writer.Write([]byte("foo"))
- writer.Close()
+ func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID())
<-st.handled
type FailThenSucceedHandler struct {
handled chan string
count int
- successhandler StubGetHandler
+ successhandler http.Handler
+ reqIDs []string
}
func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
if fh.count == 0 {
resp.WriteHeader(500)
fh.count += 1
func(kc *KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
hash,
"abc123",
"foo",
+ "",
make(chan string, 4)}
fh := FailHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 1)}
fh := FailHandler{
func (s *StandaloneSuite) TestGetFailRetry(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- st := &FailThenSucceedHandler{make(chan string, 1), 0,
- StubGetHandler{
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubGetHandler{
c,
hash,
"abc123",
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
c.Check(content, DeepEquals, []byte("foo"))
+
+ c.Logf("%q", st.reqIDs)
+ c.Assert(len(st.reqIDs) > 1, Equals, true)
+ for _, reqid := range st.reqIDs {
+ c.Check(reqid, Not(Equals), "")
+ c.Check(reqid, Equals, st.reqIDs[0])
+ }
}
func (s *StandaloneSuite) TestGetNetError(c *C) {
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
nil,
map[string]string{
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
)
r, n, uri, err := kc.Get(hash + "+K@" + uuid)
c.Check(n, Equals, int64(len(content)))
c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
}
+ {
+ loc, err := kc.LocalLocator(hash)
+ c.Check(err, Equals, nil)
+ c.Assert(len(loc) >= 32, Equals, true)
+ c.Check(loc[:32], Equals, hash[:32])
+ }
+ {
+ content := []byte("the perth county conspiracy")
+ loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content)))
+ c.Check(loc, Equals, "")
+ c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`)
+ c.Check(err, ErrorMatches, `.*HTTP 400.*`)
+ }
}
type StubProxyHandler struct {
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
}
-type FailThenSucceedPutHandler struct {
- handled chan string
- count int
- successhandler StubPutHandler
-}
-
-func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- if h.count == 0 {
- resp.WriteHeader(500)
- h.count += 1
- h.handled <- fmt.Sprintf("http://%s", req.Host)
- } else {
- h.successhandler.ServeHTTP(resp, req)
- }
-}
-
func (s *StandaloneSuite) TestPutBRetry(c *C) {
- st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
- StubPutHandler{
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubPutHandler{
c,
Md5String("foo"),
"abc123",
"foo",
+ "",
make(chan string, 5)}}
arv, _ := arvadosclient.MakeArvadosClient()