X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/1a001045ccce247a13266a5ef619ffbc07308226..df9e166a5ffc4aa79658bec1a5d552a3b413f0d8:/sdk/go/keepclient/keepclient_test.go

diff --git a/sdk/go/keepclient/keepclient_test.go b/sdk/go/keepclient/keepclient_test.go
index c1088ef8c8..cbd27d72e7 100644
--- a/sdk/go/keepclient/keepclient_test.go
+++ b/sdk/go/keepclient/keepclient_test.go
@@ -1,11 +1,12 @@
 package keepclient
 
 import (
-	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
-	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	"crypto/md5"
 	"flag"
 	"fmt"
+	"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+	"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+	"git.curoverse.com/arvados.git/sdk/go/streamer"
 	. "gopkg.in/check.v1"
 	"io"
 	"io/ioutil"
@@ -13,7 +14,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"os/exec"
 	"testing"
 )
 
@@ -44,42 +44,19 @@ func (s *ServerRequiredSuite) SetUpSuite(c *C) {
 		c.Skip("Skipping tests that require server")
 		return
 	}
-	os.Chdir(pythonDir())
-	{
-		cmd := exec.Command("python", "run_test_server.py", "start")
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			log.Fatalf("Setting up stderr pipe: %s", err)
-		}
-		go io.Copy(os.Stderr, stderr)
-		if err := cmd.Run(); err != nil {
-			panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
-		}
-	}
-	{
-		cmd := exec.Command("python", "run_test_server.py", "start_keep")
-		stderr, err := cmd.StderrPipe()
-		if err != nil {
-			log.Fatalf("Setting up stderr pipe: %s", err)
-		}
-		go io.Copy(os.Stderr, stderr)
-		if err := cmd.Run(); err != nil {
-			panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
-		}
-	}
+	arvadostest.StartAPI()
+	arvadostest.StartKeep()
 }
 
 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
-	os.Chdir(pythonDir())
-	exec.Command("python", "run_test_server.py", "stop_keep").Run()
-	exec.Command("python", "run_test_server.py", "stop").Run()
+	if *no_server {
+		return
+	}
+	arvadostest.StopKeep()
+	arvadostest.StopAPI()
 }
 
 func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
-	os.Setenv("ARVADOS_API_HOST", "localhost:3000")
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
-
 	arv, err := arvadosclient.MakeArvadosClient()
 	c.Assert(err, Equals, nil)
 
@@ -88,7 +65,7 @@ func (s *ServerRequiredSuite) TestMakeKeepClient(c *C) {
 	c.Assert(err, Equals, nil)
 	c.Check(len(kc.ServiceRoots()), Equals, 2)
 	for _, root := range kc.ServiceRoots() {
-		c.Check(root, Matches, "http://localhost:2510[\\d]")
+		c.Check(root, Matches, "http://localhost:\\d+")
 	}
 }
 
@@ -110,24 +87,22 @@ func (this StubPutHandler) ServeHTTP(resp http.ResponseWriter, req *http.Request
 	this.handled <- fmt.Sprintf("http://%s", req.Host)
 }
 
-func RunBogusKeepServer(st http.Handler, port int) (listener net.Listener, url string) {
+func RunFakeKeepServer(st http.Handler) (ks KeepServer) {
 	var err error
-	listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: port})
+	ks.listener, err = net.ListenTCP("tcp", &net.TCPAddr{Port: 0})
 	if err != nil {
-		panic(fmt.Sprintf("Could not listen on tcp port %v", port))
+		panic(fmt.Sprintf("Could not listen on any port"))
 	}
-
-	url = fmt.Sprintf("http://localhost:%d", port)
-
-	go http.Serve(listener, st)
-	return listener, url
+	ks.url = fmt.Sprintf("http://%s", ks.listener.Addr().String())
+	go http.Serve(ks.listener, st)
+	return
 }
 
 func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string,
 	io.ReadCloser, io.WriteCloser, chan uploadStatus)) {
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, _ := arvadosclient.MakeArvadosClient()
 	arv.ApiToken = "abc123"
@@ -137,7 +112,7 @@ func UploadToStubHelper(c *C, st http.Handler, f func(KeepClient, string,
 	reader, writer := io.Pipe()
 	upload_status := make(chan uploadStatus)
 
-	f(kc, url, reader, writer, upload_status)
+	f(kc, ks.url, reader, writer, upload_status)
 }
 
 func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
@@ -154,7 +129,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServer(c *C) {
 		func(kc KeepClient, url string, reader io.ReadCloser,
 			writer io.WriteCloser, upload_status chan uploadStatus) {
 
-			go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")))
+			go kc.uploadToKeepServer(url, st.expectPath, reader, upload_status, int64(len("foo")), "TestUploadToStubKeepServer")
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -186,7 +161,7 @@ func (s *StandaloneSuite) TestUploadToStubKeepServerBufferReader(c *C) {
 
 			br1 := tr.MakeStreamReader()
 
-			go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3)
+			go kc.uploadToKeepServer(url, st.expectPath, br1, upload_status, 3, "TestUploadToStubKeepServerBufferReader")
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -221,7 +196,7 @@ func (s *StandaloneSuite) TestFailedUploadToStubKeepServer(c *C) {
 		func(kc KeepClient, url string, reader io.ReadCloser,
 			writer io.WriteCloser, upload_status chan uploadStatus) {
 
-			go kc.uploadToKeepServer(url, hash, reader, upload_status, 3)
+			go kc.uploadToKeepServer(url, hash, reader, upload_status, 3, "TestFailedUploadToStubKeepServer")
 
 			writer.Write([]byte("foo"))
 			writer.Close()
@@ -240,12 +215,11 @@ type KeepServer struct {
 	url      string
 }
 
-func RunSomeFakeKeepServers(st http.Handler, n int, port int) (ks []KeepServer) {
+func RunSomeFakeKeepServers(st http.Handler, n int) (ks []KeepServer) {
 	ks = make([]KeepServer, n)
 
 	for i := 0; i < n; i += 1 {
-		boguslistener, bogusurl := RunBogusKeepServer(st, port+i)
-		ks[i] = KeepServer{boguslistener, bogusurl}
+		ks[i] = RunFakeKeepServer(st)
 	}
 
 	return ks
@@ -270,11 +244,11 @@ func (s *StandaloneSuite) TestPutB(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks := RunSomeFakeKeepServers(st, 5, 2990)
+	ks := RunSomeFakeKeepServers(st, 5)
 
-	for i := 0; i < len(ks); i += 1 {
-		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url
-		defer ks[i].listener.Close()
+	for i, k := range ks {
+		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
 	}
 
 	kc.SetServiceRoots(service_roots)
@@ -313,11 +287,11 @@ func (s *StandaloneSuite) TestPutHR(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks := RunSomeFakeKeepServers(st, 5, 2990)
+	ks := RunSomeFakeKeepServers(st, 5)
 
-	for i := 0; i < len(ks); i += 1 {
-		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = ks[i].url
-		defer ks[i].listener.Close()
+	for i, k := range ks {
+		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
+		defer k.listener.Close()
 	}
 
 	kc.SetServiceRoots(service_roots)
@@ -367,8 +341,8 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 4, 2990)
-	ks2 := RunSomeFakeKeepServers(fh, 1, 2995)
+	ks1 := RunSomeFakeKeepServers(st, 4)
+	ks2 := RunSomeFakeKeepServers(fh, 1)
 
 	for i, k := range ks1 {
 		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
@@ -391,8 +365,14 @@ func (s *StandaloneSuite) TestPutWithFail(c *C) {
 	c.Check(err, Equals, nil)
 	c.Check(phash, Equals, "")
 	c.Check(replicas, Equals, 2)
-	c.Check(<-st.handled, Equals, shuff[1])
-	c.Check(<-st.handled, Equals, shuff[2])
+
+	s1 := <-st.handled
+	s2 := <-st.handled
+
+	c.Check((s1 == shuff[1] && s2 == shuff[2]) ||
+		(s1 == shuff[2] && s2 == shuff[1]),
+		Equals,
+		true)
 }
 
 func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
@@ -417,8 +397,8 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
-	ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
+	ks1 := RunSomeFakeKeepServers(st, 1)
+	ks2 := RunSomeFakeKeepServers(fh, 4)
 
 	for i, k := range ks1 {
 		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
@@ -435,7 +415,7 @@ func (s *StandaloneSuite) TestPutWithTooManyFail(c *C) {
 
 	c.Check(err, Equals, InsufficientReplicasError)
 	c.Check(replicas, Equals, 1)
-	c.Check(<-st.handled, Matches, ".*2990")
+	c.Check(<-st.handled, Equals, ks1[0].url)
 
 	log.Printf("TestPutWithTooManyFail done")
 }
@@ -465,19 +445,19 @@ func (s *StandaloneSuite) TestGet(c *C) {
 		"abc123",
 		[]byte("foo")}
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
 	kc, _ := MakeKeepClient(&arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x":url})
+	kc.SetServiceRoots(map[string]string{"x": ks.url})
 
 	r, n, url2, err := kc.Get(hash)
 	defer r.Close()
 	c.Check(err, Equals, nil)
 	c.Check(n, Equals, int64(3))
-	c.Check(url2, Equals, fmt.Sprintf("%s/%s", url, hash))
+	c.Check(url2, Equals, fmt.Sprintf("%s/%s", ks.url, hash))
 
 	content, err2 := ioutil.ReadAll(r)
 	c.Check(err2, Equals, nil)
@@ -491,13 +471,13 @@ func (s *StandaloneSuite) TestGetFail(c *C) {
 
 	st := FailHandler{make(chan string, 1)}
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
 	kc, _ := MakeKeepClient(&arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x":url})
+	kc.SetServiceRoots(map[string]string{"x": ks.url})
 
 	r, n, url2, err := kc.Get(hash)
 	c.Check(err, Equals, BlockNotFound)
@@ -521,13 +501,13 @@ func (s *StandaloneSuite) TestChecksum(c *C) {
 
 	st := BarHandler{make(chan string, 1)}
 
-	listener, url := RunBogusKeepServer(st, 2990)
-	defer listener.Close()
+	ks := RunFakeKeepServer(st)
+	defer ks.listener.Close()
 
 	arv, err := arvadosclient.MakeArvadosClient()
 	kc, _ := MakeKeepClient(&arv)
 	arv.ApiToken = "abc123"
-	kc.SetServiceRoots(map[string]string{"x":url})
+	kc.SetServiceRoots(map[string]string{"x": ks.url})
 
 	r, n, _, err := kc.Get(barhash)
 	_, err = ioutil.ReadAll(r)
@@ -562,8 +542,8 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
-	ks2 := RunSomeFakeKeepServers(fh, 4, 2991)
+	ks1 := RunSomeFakeKeepServers(st, 1)
+	ks2 := RunSomeFakeKeepServers(fh, 4)
 
 	for i, k := range ks1 {
 		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
@@ -582,7 +562,7 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 	// the choice of block content "waz" and the UUIDs of the fake
 	// servers, so we just tried different strings until we found
 	// an example that passes this Assert.)
-	c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Matches, ".*299[1-4]")
+	c.Assert(NewRootSorter(service_roots, hash).GetSortedRoots()[0], Not(Equals), ks1[0].url)
 
 	r, n, url2, err := kc.Get(hash)
 
@@ -597,9 +577,6 @@ func (s *StandaloneSuite) TestGetWithFailures(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
-	os.Setenv("ARVADOS_API_HOST", "localhost:3000")
-	os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-	os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
 	content := []byte("TestPutGetHead")
 
 	arv, err := arvadosclient.MakeArvadosClient()
@@ -623,7 +600,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 		r, n, url2, err := kc.Get(hash)
 		c.Check(err, Equals, nil)
 		c.Check(n, Equals, int64(len(content)))
-		c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+		c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 
 		read_content, err2 := ioutil.ReadAll(r)
 		c.Check(err2, Equals, nil)
@@ -633,7 +610,7 @@ func (s *ServerRequiredSuite) TestPutGetHead(c *C) {
 		n, url2, err := kc.Ask(hash)
 		c.Check(err, Equals, nil)
 		c.Check(n, Equals, int64(len(content)))
-		c.Check(url2, Equals, fmt.Sprintf("http://localhost:25108/%s", hash))
+		c.Check(url2, Matches, fmt.Sprintf("http://localhost:\\d+/%s", hash))
 	}
 }
 
@@ -659,7 +636,7 @@ func (s *StandaloneSuite) TestPutProxy(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
+	ks1 := RunSomeFakeKeepServers(st, 1)
 
 	for i, k := range ks1 {
 		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url
@@ -690,7 +667,7 @@ func (s *StandaloneSuite) TestPutProxyInsufficientReplicas(c *C) {
 	arv.ApiToken = "abc123"
 	service_roots := make(map[string]string)
 
-	ks1 := RunSomeFakeKeepServers(st, 1, 2990)
+	ks1 := RunSomeFakeKeepServers(st, 1)
 
 	for i, k := range ks1 {
 		service_roots[fmt.Sprintf("zzzzz-bi6l4-fakefakefake%03d", i)] = k.url