+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package keepclient
import (
+ "bytes"
"crypto/md5"
- "flag"
+ "errors"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
- "git.curoverse.com/arvados.git/sdk/go/streamer"
- . "gopkg.in/check.v1"
"io"
"io/ioutil"
"log"
"net"
"net/http"
"os"
+ "strings"
"testing"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvadosclient"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ . "gopkg.in/check.v1"
)
// Gocheck boilerplate
var _ = Suite(&ServerRequiredSuite{})
var _ = Suite(&StandaloneSuite{})
-var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
-
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
// Standalone tests
type StandaloneSuite struct{}
+func (s *StandaloneSuite) SetUpTest(c *C) {
+ RefreshServiceDiscovery()
+}
+
func pythonDir() string {
cwd, _ := os.Getwd()
return fmt.Sprintf("%s/../../python/tests", cwd)
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- if *no_server {
- c.Skip("Skipping tests that require server")
- return
- }
arvadostest.StartAPI()
- arvadostest.StartKeep()
+ arvadostest.StartKeep(2, false)
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- if *no_server {
- return
- }
- arvadostest.StopKeep()
+ arvadostest.StopKeep(2)
arvadostest.StopAPI()
}
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ RefreshServiceDiscovery()
+}
+
func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
c.Check(len(kc.LocalRoots()), Equals, 2)
}
}
+func (s *ServerRequiredSuite) TestDefaultReplications(c *C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, Equals, nil)
+
+ kc, err := MakeKeepClient(arv)
+ c.Assert(kc.Want_replicas, Equals, 2)
+
+ arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
+ kc, err = MakeKeepClient(arv)
+ c.Assert(kc.Want_replicas, Equals, 3)
+
+ arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
+ kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
+ c.Assert(kc.Want_replicas, Equals, 1)
+}
+
type StubPutHandler struct {
- c *C
- expectPath string
- expectApiToken string
- expectBody string
- handled chan string
+ c *C
+ expectPath string
+ expectApiToken string
+ expectBody string
+ expectStorageClass string
+ handled chan string
}
func (sph StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
sph.c.Check(req.URL.Path, Equals, "/"+sph.expectPath)
sph.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", sph.expectApiToken))
+ sph.c.Check(req.Header.Get("X-Keep-Storage-Classes"), Equals, sph.expectStorageClass)
body, err := ioutil.ReadAll(req.Body)
sph.c.Check(err, Equals, nil)
sph.c.Check(body, DeepEquals, []byte(sph.expectBody))
func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
var err error
- ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
+ // If we don't explicitly bind it to localhost, ks.listener.Addr() will
+ // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
+ ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
if err != nil {
panic(fmt.Sprintf("Could not listen on any port"))
}
arv, _ := arvadosclient.MakeArvadosClient()
arv.ApiToken = "abc123"
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
reader, writer := io.Pipe()
upload_status := make(chan uploadStatus)
"acbd18db4cc2f85cedef654fccc4a4d8",
"abc123",
"foo",
+ "hot",
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
-
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
+ func(kc *KeepClient, url string, reader io.ReadCloser, writer io.WriteCloser, upload_status chan uploadStatus) {
+ kc.StorageClasses = []string{"hot"}
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
status := <-upload_status
c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
})
-
- log.Printf("TestUploadToStubKeepServer done")
}
func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
- log.Printf("TestUploadToStubKeepServerBufferReader")
-
st := StubPutHandler{
c,
"acbd18db4cc2f85cedef654fccc4a4d8",
"abc123",
"foo",
+ "",
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
-
- tr := streamer.AsyncStreamFromReader(512, reader)
- defer tr.Close()
-
- br1 := tr.MakeStreamReader()
-
- go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
-
- writer.Write([]byte("foo"))
- writer.Close()
+ func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, kc.getRequestID())
<-st.handled
status := <-upload_status
c.Check(status, DeepEquals, uploadStatus{nil, fmt.Sprintf("%s/%s", url, st.expectPath), 200, 1, ""})
})
-
- log.Printf("TestUploadToStubKeepServerBufferReader done")
}
type FailHandler struct {
fh.handled <- fmt.Sprintf("http://%s", req.Host)
}
-func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
- log.Printf("TestFailedUploadToStubKeepServer")
+type FailThenSucceedHandler struct {
+ handled chan string
+ count int
+ successhandler http.Handler
+ reqIDs []string
+}
+func (fh *FailThenSucceedHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ fh.reqIDs = append(fh.reqIDs, req.Header.Get("X-Request-Id"))
+ if fh.count == 0 {
+ resp.WriteHeader(500)
+ fh.count += 1
+ fh.handled <- fmt.Sprintf("http://%s", req.Host)
+ } else {
+ fh.successhandler.ServeHTTP(resp, req)
+ }
+}
+
+type Error404Handler struct {
+ handled chan string
+}
+
+func (fh Error404Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ resp.WriteHeader(404)
+ fh.handled <- fmt.Sprintf("http://%s", req.Host)
+}
+
+func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
st := FailHandler{
make(chan string)}
func(kc *KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, kc.getRequestID())
writer.Write([]byte("foo"))
writer.Close()
c.Check(status.url, Equals, fmt.Sprintf("%s/%s", url, hash))
c.Check(status.statusCode, Equals, 500)
})
- log.Printf("TestFailedUploadToStubKeepServer done")
}
type KeepServer struct {
}
func (s *StandaloneSuite) TestPutB(c *C) {
- log.Printf("TestPutB")
-
hash := Md5String("foo")
st := StubPutHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks := RunSomeFakeKeepServers(st, 5)
for i, k := range ks {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
kc.PutB([]byte("foo"))
(s1 == shuff[1] && s2 == shuff[0]),
Equals,
true)
-
- log.Printf("TestPutB done")
}
func (s *StandaloneSuite) TestPutHR(c *C) {
- log.Printf("TestPutHR")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubPutHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks := RunSomeFakeKeepServers(st, 5)
for i, k := range ks {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
reader, writer := io.Pipe()
kc.PutHR(hash, reader, 3)
shuff := NewRootSorter(kc.LocalRoots(), hash).GetSortedRoots()
- log.Print(shuff)
s1 := <-st.handled
s2 := <-st.handled
(s1 == shuff[1] && s2 == shuff[0]),
Equals,
true)
-
- log.Printf("TestPutHR done")
}
func (s *StandaloneSuite) TestPutWithFail(c *C) {
- log.Printf("TestPutWithFail")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubPutHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 4)}
fh := FailHandler{
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks1 := RunSomeFakeKeepServers(st, 4)
ks2 := RunSomeFakeKeepServers(fh, 1)
for i, k := range ks1 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
for i, k := range ks2 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
shuff := NewRootSorter(
kc.LocalRoots(), Md5String("foo")).GetSortedRoots()
+ c.Logf("%+v", shuff)
phash, replicas, err := kc.PutB([]byte("foo"))
}
func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
- log.Printf("TestPutWithTooManyFail")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubPutHandler{
hash,
"abc123",
"foo",
+ "",
make(chan string, 1)}
fh := FailHandler{
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
+ kc.Retries = 0
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks1 := RunSomeFakeKeepServers(st, 1)
ks2 := RunSomeFakeKeepServers(fh, 4)
for i, k := range ks1 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
for i, k := range ks2 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, Equals, InsufficientReplicasError)
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, ks1[0].url)
-
- log.Printf("TestPutWithTooManyFail done")
}
type StubGetHandler struct {
}
func (s *StandaloneSuite) TestGet(c *C) {
- log.Printf("TestGet")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
st := StubGetHandler{
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
- kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, n, url2, err := kc.Get(hash)
defer r.Close()
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
c.Check(content, DeepEquals, []byte("foo"))
+}
+
+func (s *StandaloneSuite) TestGet404(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
- log.Printf("TestGet done")
+ st := Error404Handler{make(chan string, 1)}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, n, url2, err := kc.Get(hash)
+ c.Check(err, Equals, BlockNotFound)
+ c.Check(n, Equals, int64(0))
+ c.Check(url2, Equals, "")
+ c.Check(r, Equals, nil)
+}
+
+func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
+ st := Error404Handler{make(chan string, 1)}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
+ c.Check(err, IsNil)
+ c.Check(n, Equals, int64(0))
+ c.Check(url2, Equals, "")
+ c.Assert(r, NotNil)
+ buf, err := ioutil.ReadAll(r)
+ c.Check(err, IsNil)
+ c.Check(buf, DeepEquals, []byte{})
}
func (s *StandaloneSuite) TestGetFail(c *C) {
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
- kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+ kc.Retries = 0
r, n, url2, err := kc.Get(hash)
- c.Check(err, Equals, BlockNotFound)
+ errNotFound, _ := err.(*ErrNotFound)
+ c.Check(errNotFound, NotNil)
+ c.Check(strings.Contains(errNotFound.Error(), "HTTP 500"), Equals, true)
+ c.Check(errNotFound.Temporary(), Equals, true)
+ c.Check(n, Equals, int64(0))
+ c.Check(url2, Equals, "")
+ c.Check(r, Equals, nil)
+}
+
+func (s *StandaloneSuite) TestGetFailRetry(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubGetHandler{
+ c,
+ hash,
+ "abc123",
+ http.StatusOK,
+ []byte("foo")}}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, n, url2, err := kc.Get(hash)
+ defer r.Close()
+ c.Check(err, Equals, nil)
+ c.Check(n, Equals, int64(3))
+ c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, []byte("foo"))
+
+ c.Logf("%q", st.reqIDs)
+ c.Assert(len(st.reqIDs) > 1, Equals, true)
+ for _, reqid := range st.reqIDs {
+ c.Check(reqid, Not(Equals), "")
+ c.Check(reqid, Equals, st.reqIDs[0])
+ }
+}
+
+func (s *StandaloneSuite) TestGetNetError(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
+
+ r, n, url2, err := kc.Get(hash)
+ errNotFound, _ := err.(*ErrNotFound)
+ c.Check(errNotFound, NotNil)
+ c.Check(strings.Contains(errNotFound.Error(), "connection refused"), Equals, true)
+ c.Check(errNotFound.Temporary(), Equals, true)
c.Check(n, Equals, int64(0))
c.Check(url2, Equals, "")
c.Check(r, Equals, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{"x": ks0.url},
+ nil,
map[string]string{uuid: ks.url})
r, n, uri, err := kc.Get(hash + "+K@" + uuid)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
+ nil,
map[string]string{
"zzzzz-bi6l4-yyyyyyyyyyyyyyy": ks0.url,
"zzzzz-bi6l4-xxxxxxxxxxxxxxx": ks0.url,
"zzzzz-bi6l4-wwwwwwwwwwwwwww": ks0.url,
- uuid: ks.url},
+ uuid: ks.url},
)
r, n, uri, err := kc.Get(hash + "+K@" + uuid)
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
+ nil,
map[string]string{uuid: ksGateway.url})
r, n, uri, err := kc.Get(hash + "+K@" + uuid)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
- kc.SetServiceRoots(map[string]string{"x": ks.url}, nil)
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, n, _, err := kc.Get(barhash)
_, err = ioutil.ReadAll(r)
content := []byte("waz")
hash := fmt.Sprintf("%x", md5.Sum(content))
- fh := FailHandler{
+ fh := Error404Handler{
make(chan string, 4)}
st := StubGetHandler{
content}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks1 := RunSomeFakeKeepServers(st, 1)
ks2 := RunSomeFakeKeepServers(fh, 4)
for i, k := range ks1 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
for i, k := range ks2 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i+len(ks1))] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+ kc.Retries = 0
// This test works only if one of the failing services is
// attempted before the succeeding service. Otherwise,
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
hash := fmt.Sprintf("%x", md5.Sum(content))
}
{
hash2, replicas, err := kc.PutB(content)
- c.Check(hash2, Equals, fmt.Sprintf("%s+%d", hash, len(content)))
+ c.Check(hash2, Matches, fmt.Sprintf(`%s\+%d\b.*`, hash, len(content)))
c.Check(replicas, Equals, 2)
c.Check(err, Equals, nil)
}
c.Check(n, Equals, int64(len(content)))
c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
}
+ {
+ loc, err := kc.LocalLocator(hash)
+ c.Check(err, Equals, nil)
+ c.Assert(len(loc) >= 32, Equals, true)
+ c.Check(loc[:32], Equals, hash[:32])
+ }
+ {
+ content := []byte("the perth county conspiracy")
+ loc, err := kc.LocalLocator(fmt.Sprintf("%x+%d+Rzaaaa-abcde@12345", md5.Sum(content), len(content)))
+ c.Check(loc, Equals, "")
+ c.Check(err, ErrorMatches, `.*HEAD .*\+R.*`)
+ c.Check(err, ErrorMatches, `.*HTTP 400.*`)
+ }
}
type StubProxyHandler struct {
}
func (s *StandaloneSuite) TestPutProxy(c *C) {
- log.Printf("TestPutProxy")
-
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
- kc.Using_proxy = true
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks1 := RunSomeFakeKeepServers(st, 1)
for i, k := range ks1 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
_, replicas, err := kc.PutB([]byte("foo"))
<-st.handled
c.Check(err, Equals, nil)
c.Check(replicas, Equals, 2)
-
- log.Printf("TestPutProxy done")
}
func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
- log.Printf("TestPutProxy")
-
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
- kc.Using_proxy = true
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
ks1 := RunSomeFakeKeepServers(st, 1)
for i, k := range ks1 {
localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
defer k.listener.Close()
}
- kc.SetServiceRoots(localRoots, nil)
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
_, replicas, err := kc.PutB([]byte("foo"))
<-st.handled
- c.Check(err, Equals, InsufficientReplicasError)
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
c.Check(replicas, Equals, 2)
-
- log.Printf("TestPutProxy done")
}
func (s *StandaloneSuite) TestMakeLocator(c *C) {
c.Check(l.Hints, DeepEquals, []string{"Aabcde@12345678"})
}
+func (s *StandaloneSuite) TestMakeLocatorPreservesUnrecognizedHints(c *C) {
+ str := "91f372a266fe2bf2823cb8ec7fda31ce+3+Unknown+Kzzzzz+Afoobar"
+ l, err := MakeLocator(str)
+ c.Check(err, Equals, nil)
+ c.Check(l.Hash, Equals, "91f372a266fe2bf2823cb8ec7fda31ce")
+ c.Check(l.Size, Equals, 3)
+ c.Check(l.Hints, DeepEquals, []string{"3", "Unknown", "Kzzzzz", "Afoobar"})
+ c.Check(l.String(), Equals, str)
+}
+
func (s *StandaloneSuite) TestMakeLocatorInvalidInput(c *C) {
_, err := MakeLocator("91f372a266fe2bf2823cb8ec7fda31c")
c.Check(err, Equals, InvalidLocatorError)
}
+
+func (s *StandaloneSuite) TestPutBWant2ReplicasWithOnlyOneWritableLocalRoot(c *C) {
+ hash := Md5String("foo")
+
+ st := StubPutHandler{
+ c,
+ hash,
+ "abc123",
+ "foo",
+ "",
+ make(chan string, 5)}
+
+ arv, _ := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+
+ kc.Want_replicas = 2
+ arv.ApiToken = "abc123"
+ localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
+
+ ks := RunSomeFakeKeepServers(st, 5)
+
+ for i, k := range ks {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ if i == 0 {
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ }
+ defer k.listener.Close()
+ }
+
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+ _, replicas, err := kc.PutB([]byte("foo"))
+
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(replicas, Equals, 1)
+
+ c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
+}
+
+func (s *StandaloneSuite) TestPutBWithNoWritableLocalRoots(c *C) {
+ hash := Md5String("foo")
+
+ st := StubPutHandler{
+ c,
+ hash,
+ "abc123",
+ "foo",
+ "",
+ make(chan string, 5)}
+
+ arv, _ := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+
+ kc.Want_replicas = 2
+ arv.ApiToken = "abc123"
+ localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
+
+ ks := RunSomeFakeKeepServers(st, 5)
+
+ for i, k := range ks {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ defer k.listener.Close()
+ }
+
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+ _, replicas, err := kc.PutB([]byte("foo"))
+
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
+ c.Check(replicas, Equals, 0)
+}
+
+type StubGetIndexHandler struct {
+ c *C
+ expectPath string
+ expectAPIToken string
+ httpStatus int
+ body []byte
+}
+
+func (h StubGetIndexHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
+ h.c.Check(req.URL.Path, Equals, h.expectPath)
+ h.c.Check(req.Header.Get("Authorization"), Equals, fmt.Sprintf("OAuth2 %s", h.expectAPIToken))
+ resp.WriteHeader(h.httpStatus)
+ resp.Header().Set("Content-Length", fmt.Sprintf("%d", len(h.body)))
+ resp.Write(h.body)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoPrefix(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index",
+ "abc123",
+ http.StatusOK,
+ []byte(hash + "+3 1443559274\n\n")}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, IsNil)
+ kc, err := MakeKeepClient(arv)
+ c.Assert(err, IsNil)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, err := kc.GetIndex("x", "")
+ c.Check(err, IsNil)
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+func (s *StandaloneSuite) TestGetIndexWithPrefix(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index/" + hash[0:3],
+ "abc123",
+ http.StatusOK,
+ []byte(hash + "+3 1443559274\n\n")}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, err := kc.GetIndex("x", hash[0:3])
+ c.Assert(err, Equals, nil)
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+func (s *StandaloneSuite) TestGetIndexIncomplete(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index/" + hash[0:3],
+ "abc123",
+ http.StatusOK,
+ []byte(hash)}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ _, err = kc.GetIndex("x", hash[0:3])
+ c.Check(err, Equals, ErrIncompleteIndex)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchServer(c *C) {
+ hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
+
+ st := StubGetIndexHandler{
+ c,
+ "/index/" + hash[0:3],
+ "abc123",
+ http.StatusOK,
+ []byte(hash)}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ _, err = kc.GetIndex("y", hash[0:3])
+ c.Check(err, Equals, ErrNoSuchKeepServer)
+}
+
+func (s *StandaloneSuite) TestGetIndexWithNoSuchPrefix(c *C) {
+ st := StubGetIndexHandler{
+ c,
+ "/index/abcd",
+ "abc123",
+ http.StatusOK,
+ []byte("\n")}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, err := kc.GetIndex("x", "abcd")
+ c.Check(err, Equals, nil)
+
+ content, err2 := ioutil.ReadAll(r)
+ c.Check(err2, Equals, nil)
+ c.Check(content, DeepEquals, st.body[0:len(st.body)-1])
+}
+
+func (s *StandaloneSuite) TestPutBRetry(c *C) {
+ st := &FailThenSucceedHandler{
+ handled: make(chan string, 1),
+ successhandler: StubPutHandler{
+ c,
+ Md5String("foo"),
+ "abc123",
+ "foo",
+ "",
+ make(chan string, 5)}}
+
+ arv, _ := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+
+ kc.Want_replicas = 2
+ arv.ApiToken = "abc123"
+ localRoots := make(map[string]string)
+ writableLocalRoots := make(map[string]string)
+
+ ks := RunSomeFakeKeepServers(st, 2)
+
+ for i, k := range ks {
+ localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ writableLocalRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ defer k.listener.Close()
+ }
+
+ kc.SetServiceRoots(localRoots, writableLocalRoots, nil)
+
+ hash, replicas, err := kc.PutB([]byte("foo"))
+
+ c.Check(err, Equals, nil)
+ c.Check(hash, Equals, "")
+ c.Check(replicas, Equals, 2)
+}
+
+func (s *ServerRequiredSuite) TestMakeKeepClientWithNonDiskTypeService(c *C) {
+ arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, Equals, nil)
+
+ // Add an additional "testblobstore" keepservice
+ blobKeepService := make(arvadosclient.Dict)
+ err = arv.Create("keep_services",
+ arvadosclient.Dict{"keep_service": arvadosclient.Dict{
+ "service_host": "localhost",
+ "service_port": "21321",
+ "service_type": "testblobstore"}},
+ &blobKeepService)
+ c.Assert(err, Equals, nil)
+ defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
+ RefreshServiceDiscovery()
+
+ // Make a keepclient and ensure that the testblobstore is included
+ kc, err := MakeKeepClient(arv)
+ c.Assert(err, Equals, nil)
+
+ // verify kc.LocalRoots
+ c.Check(len(kc.LocalRoots()), Equals, 3)
+ for _, root := range kc.LocalRoots() {
+ c.Check(root, Matches, "http://localhost:\\d+")
+ }
+ c.Assert(kc.LocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+ // verify kc.GatewayRoots
+ c.Check(len(kc.GatewayRoots()), Equals, 3)
+ for _, root := range kc.GatewayRoots() {
+ c.Check(root, Matches, "http://localhost:\\d+")
+ }
+ c.Assert(kc.GatewayRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+ // verify kc.WritableLocalRoots
+ c.Check(len(kc.WritableLocalRoots()), Equals, 3)
+ for _, root := range kc.WritableLocalRoots() {
+ c.Check(root, Matches, "http://localhost:\\d+")
+ }
+ c.Assert(kc.WritableLocalRoots()[blobKeepService["uuid"].(string)], Not(Equals), "")
+
+ c.Assert(kc.replicasPerService, Equals, 0)
+ c.Assert(kc.foundNonDiskSvc, Equals, true)
+ c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
+}
projects / arvados.git / blobdiff