X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/68c8b2056d6c72f0229cf29f999a8e0a1e4231f9..3e271eb8bc19f67465a33a69dc66a27021301fae:/services/keepproxy/keepproxy_test.go diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go index 8acf43abd4..e3b4e36b63 100644 --- a/services/keepproxy/keepproxy_test.go +++ b/services/keepproxy/keepproxy_test.go @@ -5,6 +5,7 @@ import ( "crypto/tls" "fmt" "git.curoverse.com/arvados.git/sdk/go/arvadosclient" + "git.curoverse.com/arvados.git/sdk/go/arvadostest" "git.curoverse.com/arvados.git/sdk/go/keepclient" . "gopkg.in/check.v1" "io" @@ -13,7 +14,6 @@ import ( "net/http" "net/url" "os" - "os/exec" "strings" "testing" "time" @@ -30,11 +30,6 @@ var _ = Suite(&ServerRequiredSuite{}) // Tests that require the Keep server running type ServerRequiredSuite struct{} -func pythonDir() string { - cwd, _ := os.Getwd() - return fmt.Sprintf("%s/../../sdk/python/tests", cwd) -} - // Wait (up to 1 second) for keepproxy to listen on a port. This // avoids a race condition where we hit a "connection refused" error // because we start testing the proxy too soon. @@ -57,45 +52,17 @@ func closeListener() { } func (s *ServerRequiredSuite) SetUpSuite(c *C) { - cwd, _ := os.Getwd() - defer os.Chdir(cwd) - - os.Chdir(pythonDir()) - { - cmd := exec.Command("python", "run_test_server.py", "start") - stderr, err := cmd.StderrPipe() - if err != nil { - log.Fatalf("Setting up stderr pipe: %s", err) - } - go io.Copy(os.Stderr, stderr) - if err := cmd.Run(); err != nil { - panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err)) - } - } - { - cmd := exec.Command("python", "run_test_server.py", "start_keep") - stderr, err := cmd.StderrPipe() - if err != nil { - log.Fatalf("Setting up stderr pipe: %s", err) - } - go io.Copy(os.Stderr, stderr) - if err := cmd.Run(); err != nil { - panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err)) - } - } + arvadostest.StartAPI() + arvadostest.StartKeep() +} - os.Setenv("ARVADOS_API_HOST", "localhost:3000") - os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - os.Setenv("ARVADOS_API_HOST_INSECURE", "true") +func (s *ServerRequiredSuite) SetUpTest(c *C) { + arvadostest.ResetEnv() } func (s *ServerRequiredSuite) TearDownSuite(c *C) { - cwd, _ := os.Getwd() - defer os.Chdir(cwd) - - os.Chdir(pythonDir()) - exec.Command("python", "run_test_server.py", "stop_keep").Run() - exec.Command("python", "run_test_server.py", "stop").Run() + arvadostest.StopKeep() + arvadostest.StopAPI() } func setupProxyService() { @@ -136,27 +103,37 @@ func setupProxyService() { } } -func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient { - os.Args = append(args, fmt.Sprintf("-listen=:%v", port)) - os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") - - listener = nil - go main() - time.Sleep(100 * time.Millisecond) - - os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port)) - os.Setenv("ARVADOS_API_TOKEN", token) +func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient { + if bogusClientToken { + os.Setenv("ARVADOS_API_TOKEN", "bogus-token") + } arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, Equals, nil) - kc, err := keepclient.MakeKeepClient(&arv) - c.Assert(err, Equals, nil) + kc := keepclient.KeepClient{ + Arvados: &arv, + Want_replicas: 2, + Using_proxy: true, + Client: &http.Client{}, + } + kc.SetServiceRoots(map[string]string{ + "proxy": fmt.Sprintf("http://localhost:%v", port), + }) c.Check(kc.Using_proxy, Equals, true) c.Check(len(kc.ServiceRoots()), Equals, 1) for _, root := range kc.ServiceRoots() { c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port)) } - os.Setenv("ARVADOS_KEEP_PROXY", "") log.Print("keepclient created") + if bogusClientToken { + arvadostest.ResetEnv() + } + + { + os.Args = append(args, fmt.Sprintf("-listen=:%v", port)) + listener = nil + go main() + } + return kc } @@ -164,7 +141,6 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { log.Print("TestPutAndGet start") os.Args = []string{"keepproxy", "-listen=:29950"} - os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h") listener = nil go main() time.Sleep(100 * time.Millisecond) @@ -183,7 +159,6 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(root, Equals, "http://localhost:29950") } os.Setenv("ARVADOS_EXTERNAL_CLIENT", "") - log.Print("keepclient created") waitForListener() defer closeListener() @@ -194,9 +169,25 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { { _, _, err := kc.Ask(hash) c.Check(err, Equals, keepclient.BlockNotFound) - log.Print("Ask 1") + log.Print("Finished Ask (expected BlockNotFound)") } + { + reader, _, _, err := kc.Get(hash) + c.Check(reader, Equals, nil) + c.Check(err, Equals, keepclient.BlockNotFound) + log.Print("Finished Get (expected BlockNotFound)") + } + + // Note in bug #5309 among other errors keepproxy would set + // Content-Length incorrectly on the 404 BlockNotFound response, this + // would result in a protocol violation that would prevent reuse of the + // connection, which would manifest by the next attempt to use the + // connection (in this case the PutB below) failing. So to test for + // that bug it's necessary to trigger an error response (such as + // BlockNotFound) and then do something else with the same httpClient + // connection. + { var rep int var err error @@ -204,14 +195,14 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash)) c.Check(rep, Equals, 2) c.Check(err, Equals, nil) - log.Print("PutB") + log.Print("Finished PutB (expected success)") } { blocklen, _, err := kc.Ask(hash2) c.Assert(err, Equals, nil) c.Check(blocklen, Equals, int64(3)) - log.Print("Ask 2") + log.Print("Finished Ask (expected success)") } { @@ -220,7 +211,7 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { all, err := ioutil.ReadAll(reader) c.Check(all, DeepEquals, []byte("foo")) c.Check(blocklen, Equals, int64(3)) - log.Print("Get") + log.Print("Finished Get (expected success)") } { @@ -230,7 +221,7 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`) c.Check(rep, Equals, 2) c.Check(err, Equals, nil) - log.Print("PutB zero block") + log.Print("Finished PutB zero block") } { @@ -239,7 +230,7 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { all, err := ioutil.ReadAll(reader) c.Check(all, DeepEquals, []byte("")) c.Check(blocklen, Equals, int64(0)) - log.Print("Get zero block") + log.Print("Finished Get zero block") } log.Print("TestPutAndGet done") @@ -248,12 +239,10 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) { func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { log.Print("TestPutAskGetForbidden start") - kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951) + kc := runProxy(c, []string{"keepproxy"}, 29951, true) waitForListener() defer closeListener() - log.Print("keepclient created") - hash := fmt.Sprintf("%x", md5.Sum([]byte("bar"))) { @@ -290,7 +279,7 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) { func (s *ServerRequiredSuite) TestGetDisabled(c *C) { log.Print("TestGetDisabled start") - kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952) + kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false) waitForListener() defer closeListener() @@ -330,7 +319,7 @@ func (s *ServerRequiredSuite) TestGetDisabled(c *C) { func (s *ServerRequiredSuite) TestPutDisabled(c *C) { log.Print("TestPutDisabled start") - kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953) + kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false) waitForListener() defer closeListener() @@ -346,7 +335,7 @@ func (s *ServerRequiredSuite) TestPutDisabled(c *C) { } func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { - runProxy(c, []string{"keepproxy"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29954) + runProxy(c, []string{"keepproxy"}, 29954, false) waitForListener() defer closeListener() @@ -378,7 +367,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) { } func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) { - runProxy(c, []string{"keepproxy"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29955) + runProxy(c, []string{"keepproxy"}, 29955, false) waitForListener() defer closeListener()