package main
import (
- "git.curoverse.com/arvados.git/sdk/go/keepclient"
- "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"crypto/md5"
"crypto/tls"
"fmt"
+ "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "git.curoverse.com/arvados.git/sdk/go/keepclient"
. "gopkg.in/check.v1"
"io"
"io/ioutil"
"net/http"
"net/url"
"os"
- "os/exec"
+ "strings"
"testing"
"time"
)
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
-func pythonDir() string {
- cwd, _ := os.Getwd()
- return fmt.Sprintf("%s/../../sdk/python/tests", cwd)
-}
-
// Wait (up to 1 second) for keepproxy to listen on a port. This
// avoids a race condition where we hit a "connection refused" error
// because we start testing the proxy too soon.
func waitForListener() {
- const (ms = 5)
+ const (
+ ms = 5
+ )
for i := 0; listener == nil && i < 1000; i += ms {
time.Sleep(ms * time.Millisecond)
}
}
func (s *ServerRequiredSuite) SetUpSuite(c *C) {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
+ arvadostest.StartAPI()
+ arvadostest.StartKeep()
+}
- os.Chdir(pythonDir())
- {
- cmd := exec.Command("python", "run_test_server.py", "start")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start' returned error %s", err))
- }
- }
- {
- cmd := exec.Command("python", "run_test_server.py", "start_keep")
- stderr, err := cmd.StderrPipe()
- if err != nil {
- log.Fatalf("Setting up stderr pipe: %s", err)
- }
- go io.Copy(os.Stderr, stderr)
- if err := cmd.Run(); err != nil {
- panic(fmt.Sprintf("'python run_test_server.py start_keep' returned error %s", err))
- }
- }
-
- os.Setenv("ARVADOS_API_HOST", "localhost:3000")
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
- os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
+func (s *ServerRequiredSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
}
func (s *ServerRequiredSuite) TearDownSuite(c *C) {
- cwd, _ := os.Getwd()
- defer os.Chdir(cwd)
-
- os.Chdir(pythonDir())
- exec.Command("python", "run_test_server.py", "stop_keep").Run()
- exec.Command("python", "run_test_server.py", "stop").Run()
+ arvadostest.StopKeep()
+ arvadostest.StopAPI()
}
func setupProxyService() {
}
}
-func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient {
- os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
-
- go main()
- time.Sleep(100 * time.Millisecond)
-
- os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
- os.Setenv("ARVADOS_API_TOKEN", token)
+func runProxy(c *C, args []string, port int, bogusClientToken bool) keepclient.KeepClient {
+ if bogusClientToken {
+ os.Setenv("ARVADOS_API_TOKEN", "bogus-token")
+ }
arv, err := arvadosclient.MakeArvadosClient()
- kc, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, Equals, nil)
+ kc := keepclient.KeepClient{
+ Arvados: &arv,
+ Want_replicas: 2,
+ Using_proxy: true,
+ Client: &http.Client{},
+ }
+ locals := map[string]string{
+ "proxy": fmt.Sprintf("http://localhost:%v", port),
+ }
+ writables := map[string]string{
+ "proxy": fmt.Sprintf("http://localhost:%v", port),
+ }
+ kc.SetServiceRoots(locals, nil, writables)
c.Check(kc.Using_proxy, Equals, true)
- c.Check(len(kc.ServiceRoots()), Equals, 1)
- c.Check(kc.ServiceRoots()[0], Equals, fmt.Sprintf("http://localhost:%v", port))
- c.Check(err, Equals, nil)
- os.Setenv("ARVADOS_KEEP_PROXY", "")
+ c.Check(len(kc.LocalRoots()), Equals, 1)
+ for _, root := range kc.LocalRoots() {
+ c.Check(root, Equals, fmt.Sprintf("http://localhost:%v", port))
+ }
log.Print("keepclient created")
+ if bogusClientToken {
+ arvadostest.ResetEnv()
+ }
+
+ {
+ os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
+ listener = nil
+ go main()
+ }
+
return kc
}
log.Print("TestPutAndGet start")
os.Args = []string{"keepproxy", "-listen=:29950"}
- os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ listener = nil
go main()
time.Sleep(100 * time.Millisecond)
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
arv, err := arvadosclient.MakeArvadosClient()
+ c.Assert(err, Equals, nil)
kc, err := keepclient.MakeKeepClient(&arv)
+ c.Assert(err, Equals, nil)
c.Check(kc.Arvados.External, Equals, true)
c.Check(kc.Using_proxy, Equals, true)
- c.Check(len(kc.ServiceRoots()), Equals, 1)
- c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:29950")
- c.Check(err, Equals, nil)
+ c.Check(len(kc.LocalRoots()), Equals, 1)
+ for _, root := range kc.LocalRoots() {
+ c.Check(root, Equals, "http://localhost:29950")
+ }
os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
- log.Print("keepclient created")
waitForListener()
defer closeListener()
{
_, _, err := kc.Ask(hash)
c.Check(err, Equals, keepclient.BlockNotFound)
- log.Print("Ask 1")
+ log.Print("Finished Ask (expected BlockNotFound)")
}
+ {
+ reader, _, _, err := kc.Get(hash)
+ c.Check(reader, Equals, nil)
+ c.Check(err, Equals, keepclient.BlockNotFound)
+ log.Print("Finished Get (expected BlockNotFound)")
+ }
+
+ // Note in bug #5309 among other errors keepproxy would set
+ // Content-Length incorrectly on the 404 BlockNotFound response, this
+ // would result in a protocol violation that would prevent reuse of the
+ // connection, which would manifest by the next attempt to use the
+ // connection (in this case the PutB below) failing. So to test for
+ // that bug it's necessary to trigger an error response (such as
+ // BlockNotFound) and then do something else with the same httpClient
+ // connection.
+
{
var rep int
var err error
c.Check(hash2, Matches, fmt.Sprintf(`^%s\+3(\+.+)?$`, hash))
c.Check(rep, Equals, 2)
c.Check(err, Equals, nil)
- log.Print("PutB")
+ log.Print("Finished PutB (expected success)")
}
{
blocklen, _, err := kc.Ask(hash2)
c.Assert(err, Equals, nil)
c.Check(blocklen, Equals, int64(3))
- log.Print("Ask 2")
+ log.Print("Finished Ask (expected success)")
}
{
all, err := ioutil.ReadAll(reader)
c.Check(all, DeepEquals, []byte("foo"))
c.Check(blocklen, Equals, int64(3))
- log.Print("Get")
+ log.Print("Finished Get (expected success)")
+ }
+
+ {
+ var rep int
+ var err error
+ hash2, rep, err = kc.PutB([]byte(""))
+ c.Check(hash2, Matches, `^d41d8cd98f00b204e9800998ecf8427e\+0(\+.+)?$`)
+ c.Check(rep, Equals, 2)
+ c.Check(err, Equals, nil)
+ log.Print("Finished PutB zero block")
+ }
+
+ {
+ reader, blocklen, _, err := kc.Get("d41d8cd98f00b204e9800998ecf8427e")
+ c.Assert(err, Equals, nil)
+ all, err := ioutil.ReadAll(reader)
+ c.Check(all, DeepEquals, []byte(""))
+ c.Check(blocklen, Equals, int64(0))
+ log.Print("Finished Get zero block")
}
log.Print("TestPutAndGet done")
}
func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
- log.Print("TestPutAndGet start")
+ log.Print("TestPutAskGetForbidden start")
- kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951)
+ kc := runProxy(c, []string{"keepproxy"}, 29951, true)
waitForListener()
defer closeListener()
- log.Print("keepclient created")
-
hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
{
log.Print("Get")
}
- log.Print("TestPutAndGetForbidden done")
+ log.Print("TestPutAskGetForbidden done")
}
func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
log.Print("TestGetDisabled start")
- kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952)
+ kc := runProxy(c, []string{"keepproxy", "-no-get"}, 29952, false)
waitForListener()
defer closeListener()
func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
log.Print("TestPutDisabled start")
- kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953)
+ kc := runProxy(c, []string{"keepproxy", "-no-put"}, 29953, false)
waitForListener()
defer closeListener()
log.Print("TestPutDisabled done")
}
+
+func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
+ runProxy(c, []string{"keepproxy"}, 29954, false)
+ waitForListener()
+ defer closeListener()
+
+ {
+ client := http.Client{}
+ req, err := http.NewRequest("OPTIONS",
+ fmt.Sprintf("http://localhost:29954/%x+3",
+ md5.Sum([]byte("foo"))),
+ nil)
+ req.Header.Add("Access-Control-Request-Method", "PUT")
+ req.Header.Add("Access-Control-Request-Headers", "Authorization, X-Keep-Desired-Replicas")
+ resp, err := client.Do(req)
+ c.Check(err, Equals, nil)
+ c.Check(resp.StatusCode, Equals, 200)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(string(body), Equals, "")
+ c.Check(resp.Header.Get("Access-Control-Allow-Methods"), Equals, "GET, HEAD, POST, PUT, OPTIONS")
+ c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
+ }
+
+ {
+ resp, err := http.Get(
+ fmt.Sprintf("http://localhost:29954/%x+3",
+ md5.Sum([]byte("foo"))))
+ c.Check(err, Equals, nil)
+ c.Check(resp.Header.Get("Access-Control-Allow-Headers"), Equals, "Authorization, Content-Length, Content-Type, X-Keep-Desired-Replicas")
+ c.Check(resp.Header.Get("Access-Control-Allow-Origin"), Equals, "*")
+ }
+}
+
+func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
+ runProxy(c, []string{"keepproxy"}, 29955, false)
+ waitForListener()
+ defer closeListener()
+
+ {
+ client := http.Client{}
+ req, err := http.NewRequest("POST",
+ "http://localhost:29955/",
+ strings.NewReader("qux"))
+ req.Header.Add("Authorization", "OAuth2 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
+ req.Header.Add("Content-Type", "application/octet-stream")
+ resp, err := client.Do(req)
+ c.Check(err, Equals, nil)
+ body, err := ioutil.ReadAll(resp.Body)
+ c.Check(err, Equals, nil)
+ c.Check(string(body), Equals,
+ fmt.Sprintf("%x+%d", md5.Sum([]byte("qux")), 3))
+ }
+}
+
+func (s *ServerRequiredSuite) TestStripHint(c *C) {
+ c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz", "$1"),
+ Equals,
+ "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
+ c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
+ Equals,
+ "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
+ c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz", "$1"),
+ Equals,
+ "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz")
+ c.Check(removeHint.ReplaceAllString("http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73", "$1"),
+ Equals,
+ "http://keep.zzzzz.arvadosapi.com:25107/2228819a18d3727630fa30c81853d23f+67108864+K@zzzzz-zzzzz-zzzzzzzzzzzzzzz+A37b6ab198qqqq28d903b975266b23ee711e1852c@55635f73")
+
+}
projects / arvados.git / blobdiff