+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
package keepclient
import (
+ "bytes"
"crypto/md5"
- "flag"
+ "errors"
"fmt"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/arvadostest"
- "git.curoverse.com/arvados.git/sdk/go/streamer"
- . "gopkg.in/check.v1"
"io"
"io/ioutil"
"log"
"strings"
"testing"
"time"
+
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ . "gopkg.in/check.v1"
)
// Gocheck boilerplate
var _ = Suite(&ServerRequiredSuite{})
var _ = Suite(&StandaloneSuite{})
-var no_server = flag.Bool("no-server", false, "Skip 'ServerRequireSuite'")
-
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
// Standalone tests
type StandaloneSuite struct{}
+func (s *StandaloneSuite) SetUpTest(c *C) {
+ RefreshServiceDiscovery()
+}
+
func pythonDir() string {
cwd, _ := os.Getwd()
return fmt.Sprintf("%s/../../python/tests", cwd)
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- if *no_server {
- c.Skip("Skipping tests that require server")
- return
- }
arvadostest.StartAPI()
arvadostest.StartKeep(2, false)
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- if *no_server {
- return
- }
arvadostest.StopKeep(2)
arvadostest.StopAPI()
}
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ RefreshServiceDiscovery()
+}
+
func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
c.Check(len(kc.LocalRoots()), Equals, 2)
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(kc.Want_replicas, Equals, 2)
arv.DiscoveryDoc["defaultCollectionReplication"] = 3.0
- kc, err = MakeKeepClient(&arv)
+ kc, err = MakeKeepClient(arv)
c.Assert(kc.Want_replicas, Equals, 3)
arv.DiscoveryDoc["defaultCollectionReplication"] = 1.0
- kc, err = MakeKeepClient(&arv)
+ kc, err = MakeKeepClient(arv)
+ c.Check(err, IsNil)
c.Assert(kc.Want_replicas, Equals, 1)
}
func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
var err error
- ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
+ // If we don't explicitly bind it to localhost, ks.listener.Addr() will
+ // bind to 0.0.0.0 or [::] which is not a valid address for Dial()
+ ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{IP: []byte{127, 0, 0, 1}, Port: 0})
if err != nil {
panic(fmt.Sprintf("Could not listen on any port"))
}
arv, _ := arvadosclient.MakeArvadosClient()
arv.ApiToken = "abc123"
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
reader, writer := io.Pipe()
upload_status := make(chan uploadStatus)
make(chan string)}
UploadToStubHelper(c, st,
- func(kc *KeepClient, url string, reader io.ReadCloser,
- writer io.WriteCloser, upload_status chan uploadStatus) {
-
- tr := streamer.AsyncStreamFromReader(512, reader)
- defer tr.Close()
-
- br1 := tr.MakeStreamReader()
-
- go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, 0)
-
- writer.Write([]byte("foo"))
- writer.Close()
+ func(kc *KeepClient, url string, _ io.ReadCloser, _ io.WriteCloser, upload_status chan uploadStatus) {
+ go kc.uploadToKeepServer(url, st.expectPath, bytes.NewBuffer([]byte("foo")), upload_status, 3, 0)
<-st.handled
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
make(chan string, 4)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
kc.Retries = 0
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, Equals, InsufficientReplicasError)
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, ks1[0].url)
}
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
c.Check(r, Equals, nil)
}
+func (s *StandaloneSuite) TestGetEmptyBlock(c *C) {
+ st := Error404Handler{make(chan string, 1)}
+
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
+
+ arv, err := arvadosclient.MakeArvadosClient()
+ kc, _ := MakeKeepClient(arv)
+ arv.ApiToken = "abc123"
+ kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
+
+ r, n, url2, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e+0")
+ c.Check(err, IsNil)
+ c.Check(n, Equals, int64(0))
+ c.Check(url2, Equals, "")
+ c.Assert(r, NotNil)
+ buf, err := ioutil.ReadAll(r)
+ c.Check(err, IsNil)
+ c.Check(buf, DeepEquals, []byte{})
+}
+
func (s *StandaloneSuite) TestGetFail(c *C) {
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
kc.Retries = 0
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": "http://localhost:62222"}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{"x": ks0.url},
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{
defer ksGateway.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(
map[string]string{"zzzzz-bi6l4-keepdisk0000000": ksLocal.url},
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
content}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
localRoots := make(map[string]string)
writableLocalRoots := make(map[string]string)
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
hash := fmt.Sprintf("%x", md5.Sum(content))
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
st := StubProxyHandler{make(chan string, 1)}
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 3
arv.ApiToken = "abc123"
_, replicas, err := kc.PutB([]byte("foo"))
<-st.handled
- c.Check(err, Equals, InsufficientReplicasError)
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
c.Check(replicas, Equals, 2)
}
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, Equals, InsufficientReplicasError)
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
c.Check(replicas, Equals, 1)
c.Check(<-st.handled, Equals, localRoots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", 0)])
make(chan string, 5)}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
_, replicas, err := kc.PutB([]byte("foo"))
- c.Check(err, Equals, InsufficientReplicasError)
+ c.Check(err, FitsTypeOf, InsufficientReplicasError(errors.New("")))
c.Check(replicas, Equals, 0)
}
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ c.Assert(err, IsNil)
+ kc, err := MakeKeepClient(arv)
+ c.Assert(err, IsNil)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, err := kc.GetIndex("x", "")
- c.Check(err, Equals, nil)
+ c.Check(err, IsNil)
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
r, err := kc.GetIndex("x", hash[0:3])
- c.Check(err, Equals, nil)
+ c.Assert(err, Equals, nil)
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
arv.ApiToken = "abc123"
kc.SetServiceRoots(map[string]string{"x": ks.url}, nil, nil)
make(chan string, 5)}}
arv, _ := arvadosclient.MakeArvadosClient()
- kc, _ := MakeKeepClient(&arv)
+ kc, _ := MakeKeepClient(arv)
kc.Want_replicas = 2
arv.ApiToken = "abc123"
&blobKeepService)
c.Assert(err, Equals, nil)
defer func() { arv.Delete("keep_services", blobKeepService["uuid"].(string), nil, nil) }()
+ RefreshServiceDiscovery()
// Make a keepclient and ensure that the testblobstore is included
- kc, err := MakeKeepClient(&arv)
+ kc, err := MakeKeepClient(arv)
c.Assert(err, Equals, nil)
// verify kc.LocalRoots
c.Assert(kc.replicasPerService, Equals, 0)
c.Assert(kc.foundNonDiskSvc, Equals, true)
- c.Assert(kc.Client.Timeout, Equals, 300*time.Second)
+ c.Assert(kc.httpClient().(*http.Client).Timeout, Equals, 300*time.Second)
}