package keepclient
import (
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
- "git.curoverse.com/arvados.git/sdk/go/streamer"
"crypto/md5"
"flag"
"fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/streamer"
. "gopkg.in/check.v1"
"io"
"io/ioutil"
"net"
"net/http"
"os"
- "os/exec"
"testing"
)
c.Skip("Skipping tests that require server")
return
}
- os.Chdir(pythonDir())
- {
- cmd := exec.Command("python", "run_test_server.py", "start")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
- }
- }
- {
- cmd := exec.Command("python", "run_test_server.py", "start_keep")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
- }
- }
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- os.Chdir(pythonDir())
- exec.Command("python", "run_test_server.py", "stop_keep").Run()
- exec.Command("python", "run_test_server.py", "stop").Run()
+ if *no_server {
+ return
+ }
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
}
func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
arv, err := arvadosclient.MakeArvadosClient()
c.Assert(err, Equals, nil)
c.Assert(err, Equals, nil)
c.Check(len(kc.ServiceRoots()), Equals, 2)
for _, root := range kc.ServiceRoots() {
- c.Check(root, Matches, "http://localhost:2510[\\d]")
+ c.Check(root, Matches, "http://localhost:\\d+")
}
}
-func (s *StandaloneSuite) TestShuffleServiceRoots(c *C) {
- roots := map[string]string{
- "zzzzz-bi6l4-2q7dq8becevdqfb": "http://localhost:1",
- "zzzzz-bi6l4-4gbhck2w7lq0d96": "http://localhost:2",
- "zzzzz-bi6l4-4bt69dsk0quh7ae": "http://localhost:3",
- "zzzzz-bi6l4-62w1fgd0ud2krxl": "http://localhost:4",
- }
- kc := KeepClient{}
- kc.SetServiceRoots(roots)
-
- // "foo" acbd18db4cc2f85cedef654fccc4a4d8
- foo_shuffle := []string{"http://localhost:4", "http://localhost:1", "http://localhost:3", "http://localhost:2"}
- c.Check(NewRootSorter(
- kc.ServiceRoots(), Md5String("foo")).GetSortedRoots(),
- DeepEquals, foo_shuffle)
-
- // "bar" 37b51d194a7513e45b56f6524f2d51f2
- bar_shuffle := []string{"http://localhost:3", "http://localhost:2", "http://localhost:4", "http://localhost:1"}
- c.Check(NewRootSorter(
- kc.ServiceRoots(), Md5String("bar")).GetSortedRoots(),
- DeepEquals, bar_shuffle)
-}
-
type StubPutHandler struct {
c *C
expectPath string
this.handled <- fmt.Sprintf("http://%s", req.Host)
}
-func RunBogusKeepServer(st http.Handler, port int) (listener net.Listener, url string) {
+func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
var err error
- listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: port})
+ ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
if err != nil {
- panic(fmt.Sprintf("Could not listen on tcp port %v", port))
+ panic(fmt.Sprintf("Could not listen on any port"))
}
-
- url = fmt.Sprintf("http://localhost:%d", port)
-
- go http.Serve(listener, st)
- return listener, url
+ ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
+ go http.Serve(ks.listener, st)
+ return
}
func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string,
io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
- listener, url := RunBogusKeepServer(st, 2990)
- defer listener.Close()
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
arv, _ := arvadosclient.MakeArvadosClient()
arv.ApiToken = "abc123"
reader, writer := io.Pipe()
upload_status := make(chan uploadStatus)
- f(kc, url, reader, writer, upload_status)
+ f(kc, ks.url, reader, writer, upload_status)
}
func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
func(kc KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")))
+ go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
writer.Write([]byte("foo"))
writer.Close()
br1 := tr.MakeStreamReader()
- go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3)
+ go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
writer.Write([]byte("foo"))
writer.Close()
func(kc KeepClient, url string, reader io.ReadCloser,
writer io.WriteCloser, upload_status chan uploadStatus) {
- go kc.uploadToKeepServer(url, hash, reader, upload_status, 3)
+ go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
writer.Write([]byte("foo"))
writer.Close()
url string
}
-func RunSomeFakeKeepServers(st http.Handler, n int, port int) (ks []KeepServer) {
+func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
ks = make([]KeepServer, n)
for i := 0; i < n; i += 1 {
- boguslistener, bogusurl := RunBogusKeepServer(st, port+i)
- ks[i] = KeepServer{boguslistener, bogusurl}
+ ks[i] = RunFakeKeepServer(st)
}
return ks
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks := RunSomeFakeKeepServers(st, 5, 2990)
+ ks := RunSomeFakeKeepServers(st, 5)
- for i := 0; i < len(ks); i += 1 {
- service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url
- defer ks[i].listener.Close()
+ for i, k := range ks {
+ service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ defer k.listener.Close()
}
kc.SetServiceRoots(service_roots)
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks := RunSomeFakeKeepServers(st, 5, 2990)
+ ks := RunSomeFakeKeepServers(st, 5)
- for i := 0; i < len(ks); i += 1 {
- service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url
- defer ks[i].listener.Close()
+ for i, k := range ks {
+ service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+ defer k.listener.Close()
}
kc.SetServiceRoots(service_roots)
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks1 := RunSomeFakeKeepServers(st, 4, 2990)
- ks2 := RunSomeFakeKeepServers(fh, 1, 2995)
+ ks1 := RunSomeFakeKeepServers(st, 4)
+ ks2 := RunSomeFakeKeepServers(fh, 1)
for i, k := range ks1 {
service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
c.Check(err, Equals, nil)
c.Check(phash, Equals, "")
c.Check(replicas, Equals, 2)
- c.Check(<-st.handled, Equals, shuff[1])
- c.Check(<-st.handled, Equals, shuff[2])
+
+ s1 := <-st.handled
+ s2 := <-st.handled
+
+ c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
+ (s1 == shuff[2] && s2 == shuff[1]),
+ Equals,
+ true)
}
func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks1 := RunSomeFakeKeepServers(st, 1, 2990)
- ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
+ ks1 := RunSomeFakeKeepServers(st, 1)
+ ks2 := RunSomeFakeKeepServers(fh, 4)
for i, k := range ks1 {
service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
c.Check(err, Equals, InsufficientReplicasError)
c.Check(replicas, Equals, 1)
- c.Check(<-st.handled, Matches, ".*2990")
+ c.Check(<-st.handled, Equals, ks1[0].url)
log.Printf("TestPutWithTooManyFail done")
}
"abc123",
[]byte("foo")}
- listener, url := RunBogusKeepServer(st, 2990)
- defer listener.Close()
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(&arv)
arv.ApiToken = "abc123"
- kc.SetServiceRoots(map[string]string{"x":url})
+ kc.SetServiceRoots(map[string]string{"x": ks.url})
r, n, url2, err := kc.Get(hash)
defer r.Close()
c.Check(err, Equals, nil)
c.Check(n, Equals, int64(3))
- c.Check(url2, Equals, fmt.Sprintf("%s/%s", url, hash))
+ c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
st := FailHandler{make(chan string, 1)}
- listener, url := RunBogusKeepServer(st, 2990)
- defer listener.Close()
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(&arv)
arv.ApiToken = "abc123"
- kc.SetServiceRoots(map[string]string{"x":url})
+ kc.SetServiceRoots(map[string]string{"x": ks.url})
r, n, url2, err := kc.Get(hash)
c.Check(err, Equals, BlockNotFound)
st := BarHandler{make(chan string, 1)}
- listener, url := RunBogusKeepServer(st, 2990)
- defer listener.Close()
+ ks := RunFakeKeepServer(st)
+ defer ks.listener.Close()
arv, err := arvadosclient.MakeArvadosClient()
kc, _ := MakeKeepClient(&arv)
arv.ApiToken = "abc123"
- kc.SetServiceRoots(map[string]string{"x":url})
+ kc.SetServiceRoots(map[string]string{"x": ks.url})
r, n, _, err := kc.Get(barhash)
_, err = ioutil.ReadAll(r)
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks1 := RunSomeFakeKeepServers(st, 1, 2990)
- ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
+ ks1 := RunSomeFakeKeepServers(st, 1)
+ ks2 := RunSomeFakeKeepServers(fh, 4)
for i, k := range ks1 {
service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
// the choice of block content "waz" and the UUIDs of the fake
// servers, so we just tried different strings until we found
// an example that passes this Assert.)
- c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Matches, ".*299[1-4]")
+ c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
r, n, url2, err := kc.Get(hash)
}
func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
content := []byte("TestPutGetHead")
arv, err := arvadosclient.MakeArvadosClient()
r, n, url2, err := kc.Get(hash)
c.Check(err, Equals, nil)
c.Check(n, Equals, int64(len(content)))
- c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+ c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
read_content, err2 := ioutil.ReadAll(r)
c.Check(err2, Equals, nil)
n, url2, err := kc.Ask(hash)
c.Check(err, Equals, nil)
c.Check(n, Equals, int64(len(content)))
- c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+ c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
}
}
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks1 := RunSomeFakeKeepServers(st, 1, 2990)
+ ks1 := RunSomeFakeKeepServers(st, 1)
for i, k := range ks1 {
service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
arv.ApiToken = "abc123"
service_roots := make(map[string]string)
- ks1 := RunSomeFakeKeepServers(st, 1, 2990)
+ ks1 := RunSomeFakeKeepServers(st, 1)
for i, k := range ks1 {
service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
projects / arvados.git / blobdiff