From: Tom Clegg Date: Fri, 30 Oct 2015 18:30:13 +0000 (-0400) Subject: Merge branch '5824-keep-web' into 5824-keep-web-workbench X-Git-Tag: 1.1.0~1259^2~19 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/a4acb3ae95b2fc7f4b5f1e174c910a54cc6681da Merge branch '5824-keep-web' into 5824-keep-web-workbench Conflicts: sdk/python/tests/run_test_server.py services/keepproxy/keepproxy_test.go --- a4acb3ae95b2fc7f4b5f1e174c910a54cc6681da diff --cc sdk/python/tests/run_test_server.py index 591b500cfe,d325b4eb6e..f8f8b18d76 --- a/sdk/python/tests/run_test_server.py +++ b/sdk/python/tests/run_test_server.py @@@ -496,7 -471,9 +502,10 @@@ def run_nginx() '-g', 'pid '+_pidfile('nginx')+';', '-c', conffile], env=env, stdin=open('/dev/null'), stdout=sys.stderr) + cat_access = subprocess.Popen( + ['cat', nginxconf['ACCESSLOG']], + stdout=sys.stderr) + _setport('keep-web-ssl', nginxconf['KEEPWEBSSLPORT']) _setport('keepproxy-ssl', nginxconf['KEEPPROXYSSLPORT']) _setport('arv-git-httpd-ssl', nginxconf['GITSSLPORT']) diff --cc services/keepproxy/keepproxy_test.go index f350e0b657,2c75ec1616..e4f09b4e74 --- a/services/keepproxy/keepproxy_test.go +++ b/services/keepproxy/keepproxy_test.go @@@ -28,8 -30,12 +28,14 @@@ var _ = Suite(&ServerRequiredSuite{} // Tests that require the Keep server running type ServerRequiredSuite struct{} + // Gocheck boilerplate + var _ = Suite(&NoKeepServerSuite{}) + + // Test with no keepserver to simulate errors + type NoKeepServerSuite struct{} + +var TestProxyUUID = "zzzzz-bi6l4-lrixqc4fxofbmzz" + // Wait (up to 1 second) for keepproxy to listen on a port. This // avoids a race condition where we hit a "connection refused" error // because we start testing the proxy too soon. @@@ -65,27 -83,80 +83,27 @@@ func (s *NoKeepServerSuite) TearDownSui arvadostest.StopAPI() } -func setupProxyService() { - - client := &http.Client{Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} - - var req *http.Request - var err error - if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil { - panic(err.Error()) - } - req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN"))) - - reader, writer := io.Pipe() - - req.Body = reader - - go func() { - data := url.Values{} - data.Set("keep_service", `{ - "service_host": "localhost", - "service_port": 29950, - "service_ssl_flag": false, - "service_type": "proxy" -}`) - - writer.Write([]byte(data.Encode())) - writer.Close() - }() - - var resp *http.Response - if resp, err = client.Do(req); err != nil { - panic(err.Error()) - } - if resp.StatusCode != 200 { - panic(resp.Status) - } -} +func runProxy(c *C, args []string, bogusClientToken bool) *keepclient.KeepClient { + args = append([]string{"keepproxy"}, args...) + os.Args = append(args, "-listen=:0") + listener = nil + go main() + waitForListener() -func runProxy(c *C, args []string, port int, bogusClientToken bool) *keepclient.KeepClient { - if bogusClientToken { - os.Setenv("ARVADOS_API_TOKEN", "bogus-token") - } arv, err := arvadosclient.MakeArvadosClient() c.Assert(err, Equals, nil) - kc := keepclient.KeepClient{ - Arvados: &arv, - Want_replicas: 2, - Using_proxy: true, - Client: &http.Client{}, - } - locals := map[string]string{ - "proxy": fmt.Sprintf("http://localhost:%v", port), - } - writableLocals := map[string]string{ - "proxy": fmt.Sprintf("http://localhost:%v", port), - } - kc.SetServiceRoots(locals, writableLocals, nil) - c.Check(kc.Using_proxy, Equals, true) - c.Check(len(kc.LocalRoots()), Equals, 1) - for _, root := range kc.LocalRoots() { - c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port)) - } - log.Print("keepclient created") if bogusClientToken { - arvadostest.ResetEnv() + arv.ApiToken = "bogus-token" } - - { - os.Args = append(args, fmt.Sprintf("-listen=:%v", port)) - listener = nil - go main() + kc := keepclient.New(&arv) + sr := map[string]string{ + TestProxyUUID: "http://" + listener.Addr().String(), } + kc.SetServiceRoots(sr, sr, sr) + kc.Arvados.External = true + kc.Using_proxy = true - return kc + return &kc } func (s *ServerRequiredSuite) TestPutAskGet(c *C) { @@@ -370,6 -500,87 +400,87 @@@ func (s *ServerRequiredSuite) TestGetIn } // GetIndex with invalid prefix - _, err = kc.GetIndex("proxy", "xyz") + _, err = kc.GetIndex(TestProxyUUID, "xyz") c.Assert((err != nil), Equals, true) } + + func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) { + kc := runProxy(c, []string{"keepproxy"}, 28852, false) + waitForListener() + defer closeListener() + + // Put a test block + hash, rep, err := kc.PutB([]byte("foo")) + c.Check(err, Equals, nil) + c.Check(rep, Equals, 2) + + for _, token := range []string{ + "nosuchtoken", + "2ym314ysp27sk7h943q6vtc378srb06se3pq6ghurylyf3pdmx", // expired + } { + // Change token to given bad token + kc.Arvados.ApiToken = token + + // Ask should result in error + _, _, err = kc.Ask(hash) + c.Check(err, NotNil) + errNotFound, _ := err.(keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, false) + c.Assert(strings.Contains(err.Error(), "HTTP 403"), Equals, true) + + // Get should result in error + _, _, _, err = kc.Get(hash) + c.Check(err, NotNil) + errNotFound, _ = err.(keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, false) + c.Assert(strings.Contains(err.Error(), "HTTP 403 \"Missing or invalid Authorization header\""), Equals, true) + } + } + + func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) { + arv, err := arvadosclient.MakeArvadosClient() + c.Assert(err, Equals, nil) + + // keepclient with no such keep server + kc := keepclient.New(&arv) + locals := map[string]string{ + "proxy": "http://localhost:12345", + } + kc.SetServiceRoots(locals, nil, nil) + + // Ask should result in temporary connection refused error + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + _, _, err = kc.Ask(hash) + c.Check(err, NotNil) + errNotFound, _ := err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true) + + // Get should result in temporary connection refused error + _, _, _, err = kc.Get(hash) + c.Check(err, NotNil) + errNotFound, _ = err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Assert(strings.Contains(err.Error(), "connection refused"), Equals, true) + } + + func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) { + kc := runProxy(c, []string{"keepproxy"}, 29999, false) + waitForListener() + defer closeListener() + + // Ask should result in temporary connection refused error + hash := fmt.Sprintf("%x", md5.Sum([]byte("foo"))) + _, _, err := kc.Ask(hash) + c.Check(err, NotNil) + errNotFound, _ := err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true) + + // Get should result in temporary connection refused error + _, _, _, err = kc.Get(hash) + c.Check(err, NotNil) + errNotFound, _ = err.(*keepclient.ErrNotFound) + c.Check(errNotFound.Temporary(), Equals, true) + c.Assert(strings.Contains(err.Error(), "HTTP 502"), Equals, true) + }