import (
"bytes"
+ "context"
"crypto/md5"
- "errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
+ "sync"
"testing"
"time"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
. "gopkg.in/check.v1"
)
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopKeep(2)
- arvadostest.StopAPI()
}
func (s *ServerRequiredSuite) SetUpTest(c *C) {
}
}
+func (s *ServerRequiredSuite) TestDefaultStorageClasses(c *C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, IsNil)
+
+ cc, err := arv.ClusterConfig("StorageClasses")
+ c.Assert(err, IsNil)
+ c.Assert(cc, NotNil)
+ c.Assert(cc.(map[string]interface{})["default"], NotNil)
+
+ kc := New(arv)
+ c.Assert(kc.DefaultStorageClasses, DeepEquals, []string{"default"})
+}
+
func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
- c.Assert(err, Equals, nil)
+ c.Assert(err, IsNil)
kc, err := MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 2)
arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 3)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 1)
}
type StubPutHandler struct {
- c *C
- expectPath string
- expectApiToken string
- expectBody string
- handled chan string
-}
-
-func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ c *C
+ expectPath string
+ expectAPIToken string
+ expectBody string
+ expectStorageClass string
+ returnStorageClasses string
+ handled chan string
+ requests []*http.Request
+ mtx sync.Mutex
+}
+
+func (sph *StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ sph.mtx.Lock()
+ sph.requests = append(sph.requests, req)
+ sph.mtx.Unlock()
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
- sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectAPIToken))
+ if sph.expectStorageClass != "*" {
+ sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
+ }
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
+ resp.Header().Set("X-Keep-Replicas-Stored", "1")
+ if sph.returnStorageClasses != "" {
+ resp.Header().Set("X-Keep-Storage-Classes-Confirmed", sph.returnStorageClasses)
+ }
resp.WriteHeader(200)
sph.handled <- fmt.Sprintf("http://%s", req.Host)
}
// bind to 0.0.0.0 or [::] which is not a valid address for Dial()
ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
if err != nil {
- panic(fmt.Sprintf("Could not listen on any port"))
+ panic("Could not listen on any port")
}
ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
go http.Serve(ks.listener, st)
kc, _ := MakeKeepClient(arv)
reader, writer := io.Pipe()
- upload_status := make(chan uploadStatus)
+ uploadStatusChan := make(chan uploadStatus)
- f(kc, ks.url, reader, writer, upload_status)
+ f(kc, ks.url, reader, writer, uploadStatusChan)
}
func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
log.Printf("TestUploadToStubKeepServer")
- st := StubPutHandler{
- c,
- "acbd18db4cc2f85cedef654fccc4a4d8",
- "abc123",
- "foo",
- make(chan string)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: "acbd18db4cc2f85cedef654fccc4a4d8",
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "",
+ returnStorageClasses: "default=1",
+ handled: make(chan string),
+ }
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
-
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), 0)
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
<-st.handled
- status := <-upload_status
- c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
+ status := <-uploadStatusChan
+ c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""})
})
}
func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
- st := StubPutHandler{
- c,
- "acbd18db4cc2f85cedef654fccc4a4d8",
- "abc123",
- "foo",
- make(chan string)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: "acbd18db4cc2f85cedef654fccc4a4d8",
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "",
+ returnStorageClasses: "default=1",
+ handled: make(chan string),
+ }
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0)
+ func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, uploadStatusChan chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, nil, bytes.NewBuffer([]byte("foo")), uploadStatusChan, 3, kc.getRequestID())
<-st.handled
- status := <-upload_status
- c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
+ status := <-uploadStatusChan
+ c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, map[string]int{"default": 1}, ""})
+ })
+}
+
+func (s *StandaloneSuite) TestUploadWithStorageClasses(c *C) {
+ for _, trial := range []struct {
+ respHeader string
+ expectMap map[string]int
+ }{
+ {"", nil},
+ {"foo=1", map[string]int{"foo": 1}},
+ {" foo=1 , bar=2 ", map[string]int{"foo": 1, "bar": 2}},
+ {" =foo=1 ", nil},
+ {"foo", nil},
+ } {
+ st := &StubPutHandler{
+ c: c,
+ expectPath: "acbd18db4cc2f85cedef654fccc4a4d8",
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "",
+ returnStorageClasses: trial.respHeader,
+ handled: make(chan string),
+ }
+
+ UploadToStubHelper(c, st,
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, nil, reader, uploadStatusChan, len("foo"), kc.getRequestID())
+
+ writer.Write([]byte("foo"))
+ writer.Close()
+
+ <-st.handled
+ status := <-uploadStatusChan
+ c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, trial.expectMap, ""})
+ })
+ }
+}
+
+func (s *StandaloneSuite) TestPutWithoutStorageClassesClusterSupport(c *C) {
+ nServers := 5
+ for _, trial := range []struct {
+ replicas int
+ clientClasses []string
+ putClasses []string
+ minRequests int
+ maxRequests int
+ success bool
+ }{
+ // Talking to an older cluster (no default storage classes exported
+ // config) and no other additional storage classes requirements.
+ {1, nil, nil, 1, 1, true},
+ {2, nil, nil, 2, 2, true},
+ {3, nil, nil, 3, 3, true},
+ {nServers*2 + 1, nil, nil, nServers, nServers, false},
+
+ {1, []string{"class1"}, nil, 1, 1, true},
+ {2, []string{"class1"}, nil, 2, 2, true},
+ {3, []string{"class1"}, nil, 3, 3, true},
+ {1, []string{"class1", "class2"}, nil, 1, 1, true},
+ {nServers*2 + 1, []string{"class1"}, nil, nServers, nServers, false},
+
+ {1, nil, []string{"class1"}, 1, 1, true},
+ {2, nil, []string{"class1"}, 2, 2, true},
+ {3, nil, []string{"class1"}, 3, 3, true},
+ {1, nil, []string{"class1", "class2"}, 1, 1, true},
+ {nServers*2 + 1, nil, []string{"class1"}, nServers, nServers, false},
+ } {
+ c.Logf("%+v", trial)
+ st := &StubPutHandler{
+ c: c,
+ expectPath: "acbd18db4cc2f85cedef654fccc4a4d8",
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "*",
+ returnStorageClasses: "", // Simulate old cluster without SC keep support
+ handled: make(chan string, 100),
+ }
+ ks := RunSomeFakeKeepServers(st, nServers)
+ arv, _ := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ kc.Want_replicas = trial.replicas
+ kc.StorageClasses = trial.clientClasses
+ kc.DefaultStorageClasses = nil // Simulate an old cluster without SC defaults
+ arv.ApiToken = "abc123"
+ localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
+ for i, k := range ks {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ defer k.listener.Close()
+ }
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+ _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Data: []byte("foo"),
+ StorageClasses: trial.putClasses,
})
+ if trial.success {
+ c.Check(err, IsNil)
+ } else {
+ c.Check(err, NotNil)
+ }
+ c.Check(len(st.handled) >= trial.minRequests, Equals, true, Commentf("len(st.handled)==%d, trial.minRequests==%d", len(st.handled), trial.minRequests))
+ c.Check(len(st.handled) <= trial.maxRequests, Equals, true, Commentf("len(st.handled)==%d, trial.maxRequests==%d", len(st.handled), trial.maxRequests))
+ if trial.clientClasses == nil && trial.putClasses == nil {
+ c.Check(st.requests[0].Header.Get("X-Keep-Storage-Classes"), Equals, "")
+ }
+ }
+}
+
+func (s *StandaloneSuite) TestPutWithStorageClasses(c *C) {
+ nServers := 5
+ for _, trial := range []struct {
+ replicas int
+ defaultClasses []string
+ clientClasses []string // clientClasses takes precedence over defaultClasses
+ putClasses []string // putClasses takes precedence over clientClasses
+ minRequests int
+ maxRequests int
+ success bool
+ }{
+ {1, []string{"class1"}, nil, nil, 1, 1, true},
+ {2, []string{"class1"}, nil, nil, 1, 2, true},
+ {3, []string{"class1"}, nil, nil, 2, 3, true},
+ {1, []string{"class1", "class2"}, nil, nil, 1, 1, true},
+
+ // defaultClasses doesn't matter when any of the others is specified.
+ {1, []string{"class1"}, []string{"class1"}, nil, 1, 1, true},
+ {2, []string{"class1"}, []string{"class1"}, nil, 1, 2, true},
+ {3, []string{"class1"}, []string{"class1"}, nil, 2, 3, true},
+ {1, []string{"class1"}, []string{"class1", "class2"}, nil, 1, 1, true},
+ {3, []string{"class1"}, nil, []string{"class1"}, 2, 3, true},
+ {1, []string{"class1"}, nil, []string{"class1", "class2"}, 1, 1, true},
+ {1, []string{"class1"}, []string{"class404"}, []string{"class1", "class2"}, 1, 1, true},
+ {1, []string{"class1"}, []string{"class1"}, []string{"class404", "class2"}, nServers, nServers, false},
+ {nServers*2 + 1, []string{}, []string{"class1"}, nil, nServers, nServers, false},
+ {1, []string{"class1"}, []string{"class404"}, nil, nServers, nServers, false},
+ {1, []string{"class1"}, []string{"class1", "class404"}, nil, nServers, nServers, false},
+ {1, []string{"class1"}, nil, []string{"class1", "class404"}, nServers, nServers, false},
+ } {
+ c.Logf("%+v", trial)
+ st := &StubPutHandler{
+ c: c,
+ expectPath: "acbd18db4cc2f85cedef654fccc4a4d8",
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "*",
+ returnStorageClasses: "class1=2, class2=2",
+ handled: make(chan string, 100),
+ }
+ ks := RunSomeFakeKeepServers(st, nServers)
+ arv, _ := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ kc.Want_replicas = trial.replicas
+ kc.StorageClasses = trial.clientClasses
+ kc.DefaultStorageClasses = trial.defaultClasses
+ arv.ApiToken = "abc123"
+ localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
+ for i, k := range ks {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ defer k.listener.Close()
+ }
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+ _, err := kc.BlockWrite(context.Background(), arvados.BlockWriteOptions{
+ Data: []byte("foo"),
+ StorageClasses: trial.putClasses,
+ })
+ if trial.success {
+ c.Check(err, IsNil)
+ } else {
+ c.Check(err, NotNil)
+ }
+ c.Check(len(st.handled) >= trial.minRequests, Equals, true, Commentf("len(st.handled)==%d, trial.minRequests==%d", len(st.handled), trial.minRequests))
+ c.Check(len(st.handled) <= trial.maxRequests, Equals, true, Commentf("len(st.handled)==%d, trial.maxRequests==%d", len(st.handled), trial.maxRequests))
+ if !trial.success && trial.replicas == 1 && c.Check(len(st.requests) >= 2, Equals, true) {
+ // Max concurrency should be 1. First request
+ // should have succeeded for class1. Second
+ // request should only ask for class404.
+ c.Check(st.requests[1].Header.Get("X-Keep-Storage-Classes"), Equals, "class404")
+ }
+ }
}
type FailHandler struct {
type FailThenSucceedHandler struct {
handled chan string
count int
- successhandler StubGetHandler
+ successhandler http.Handler
+ reqIDs []string
}
func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
if fh.count == 0 {
resp.WriteHeader(500)
- fh.count += 1
+ fh.count++
fh.handled <- fmt.Sprintf("http://%s", req.Host)
} else {
fh.successhandler.ServeHTTP(resp, req)
UploadToStubHelper(c, st,
func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
+ writer io.WriteCloser, uploadStatusChan chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, 0)
+ go kc.uploadToKeepServer(url, hash, nil, reader, uploadStatusChan, 3, kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
<-st.handled
- status := <-upload_status
+ status := <-uploadStatusChan
c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
c.Check(status.statusCode, Equals, 500)
})
func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
ks = make([]KeepServer, n)
- for i := 0; i < n; i += 1 {
+ for i := 0; i < n; i++ {
ks[i] = RunFakeKeepServer(st)
}
func (s *StandaloneSuite) TestPutB(c *C) {
hash := Md5String("foo")
- st := StubPutHandler{
- c,
- hash,
- "abc123",
- "foo",
- make(chan string, 5)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: hash,
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "default",
+ returnStorageClasses: "",
+ handled: make(chan string, 5),
+ }
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)
func (s *StandaloneSuite) TestPutHR(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- st := StubPutHandler{
- c,
- hash,
- "abc123",
- "foo",
- make(chan string, 5)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: hash,
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "default",
+ returnStorageClasses: "",
+ handled: make(chan string, 5),
+ }
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)
func (s *StandaloneSuite) TestPutWithFail(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- st := StubPutHandler{
- c,
- hash,
- "abc123",
- "foo",
- make(chan string, 4)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: hash,
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "default",
+ returnStorageClasses: "",
+ handled: make(chan string, 4),
+ }
fh := FailHandler{
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- st := StubPutHandler{
- c,
- hash,
- "abc123",
- "foo",
- make(chan string, 1)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: hash,
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "default",
+ returnStorageClasses: "",
+ handled: make(chan string, 1),
+ }
fh := FailHandler{
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, ks1[0].url)
}
type StubGetHandler struct {
c *C
expectPath string
- expectApiToken string
+ expectAPIToken string
httpStatus int
body []byte
}
func (sgh StubGetHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sgh.c.Check(req.URL.Path, Equals, "/"+sgh.expectPath)
- sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectApiToken))
+ sgh.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sgh.expectAPIToken))
resp.WriteHeader(sgh.httpStatus)
resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(sgh.body)))
resp.Write(sgh.body)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
func (s *StandaloneSuite) TestGetFailRetry(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- st := &FailThenSucceedHandler{make(chan string, 1), 0,
- StubGetHandler{
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubGetHandler{
c,
hash,
"abc123",
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
c.Check(content, DeepEquals, []byte("foo"))
+
+ c.Logf("%q", st.reqIDs)
+ c.Assert(len(st.reqIDs) > 1, Equals, true)
+ for _, reqid := range st.reqIDs {
+ c.Check(reqid, Not(Equals), "")
+ c.Check(reqid, Equals, st.reqIDs[0])
+ }
}
func (s *StandaloneSuite) TestGetNetError(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
nil,
map[string]string{
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
)
r, n, uri, err := kc.Get(hash + "+K@" + uuid)
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
handled chan string
}
-func (this BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h BarHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
resp.Write([]byte("bar"))
- this.handled <- fmt.Sprintf("http://%s", req.Host)
+ h.handled <- fmt.Sprintf("http://%s", req.Host)
}
func (s *StandaloneSuite) TestChecksum(c *C) {
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, n, _, err := kc.Get(barhash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, nil)
<-st.handled
r, n, _, err = kc.Get(foohash)
+ c.Check(err, IsNil)
_, err = ioutil.ReadAll(r)
c.Check(n, Equals, int64(3))
c.Check(err, Equals, BadChecksum)
content}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
c.Check(n, Equals, int64(3))
c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks1[0].url, hash))
- read_content, err2 := ioutil.ReadAll(r)
+ readContent, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
- c.Check(read_content, DeepEquals, content)
+ c.Check(readContent, DeepEquals, content)
}
func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
c.Check(n, Equals, int64(len(content)))
c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
- read_content, err2 := ioutil.ReadAll(r)
+ readContent, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
- c.Check(read_content, DeepEquals, content)
+ c.Check(readContent, DeepEquals, content)
}
{
n, url2, err := kc.Ask(hash)
c.Check(n, Equals, int64(len(content)))
c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
}
+ {
+ loc, err := kc.LocalLocator(hash)
+ c.Check(err, Equals, nil)
+ c.Assert(len(loc) >= 32, Equals, true)
+ c.Check(loc[:32], Equals, hash[:32])
+ }
+ {
+ content := []byte("the perth county conspiracy")
+ loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content)))
+ c.Check(loc, Equals, "")
+ c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`)
+ c.Check(err, ErrorMatches, `.*HTTP 400.*`)
+ }
}
type StubProxyHandler struct {
handled chan string
}
-func (this StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+func (h StubProxyHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
resp.Header().Set("X-Keep-Replicas-Stored", "2")
- this.handled <- fmt.Sprintf("http://%s", req.Host)
+ h.handled <- fmt.Sprintf("http://%s", req.Host)
}
func (s *StandaloneSuite) TestPutProxy(c *C) {
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
_, replicas, err := kc.PutB([]byte("foo"))
<-st.handled
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 2)
}
func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
hash := Md5String("foo")
- st := StubPutHandler{
- c,
- hash,
- "abc123",
- "foo",
- make(chan string, 5)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: hash,
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "default",
+ returnStorageClasses: "",
+ handled: make(chan string, 5),
+ }
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
hash := Md5String("foo")
- st := StubPutHandler{
- c,
- hash,
- "abc123",
- "foo",
- make(chan string, 5)}
+ st := &StubPutHandler{
+ c: c,
+ expectPath: hash,
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "",
+ returnStorageClasses: "",
+ handled: make(chan string, 5),
+ }
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(err, FitsTypeOf, InsufficientReplicasError{})
c.Check(replicas, Equals, 0)
}
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
+ c.Check(err, IsNil)
kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
}
-type FailThenSucceedPutHandler struct {
- handled chan string
- count int
- successhandler StubPutHandler
-}
-
-func (h *FailThenSucceedPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
- if h.count == 0 {
- resp.WriteHeader(500)
- h.count += 1
- h.handled <- fmt.Sprintf("http://%s", req.Host)
- } else {
- h.successhandler.ServeHTTP(resp, req)
- }
-}
-
func (s *StandaloneSuite) TestPutBRetry(c *C) {
- st := &FailThenSucceedPutHandler{make(chan string, 1), 0,
- StubPutHandler{
- c,
- Md5String("foo"),
- "abc123",
- "foo",
- make(chan string, 5)}}
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: &StubPutHandler{
+ c: c,
+ expectPath: Md5String("foo"),
+ expectAPIToken: "abc123",
+ expectBody: "foo",
+ expectStorageClass: "default",
+ returnStorageClasses: "",
+ handled: make(chan string, 5),
+ },
+ }
arv, _ := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(arv)