services/keep/src/arvados.org/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "arvados.org/keepclient"
   5         "crypto/md5"
   6         "crypto/tls"
   7         "fmt"
   8         . "gopkg.in/check.v1"
   9         "io"
  10         "io/ioutil"
  11         "log"
  12         "net/http"
  13         "net/url"
  14         "os"
  15         "os/exec"
  16         "strings"
  17         "testing"
  18         "time"
  19 )
  20
  21 // Gocheck boilerplate
  22 func Test(t *testing.T) {
  23         TestingT(t)
  24 }
  25
  26 // Gocheck boilerplate
  27 var _ = Suite(&ServerRequiredSuite{})
  28
  29 // Tests that require the Keep server running
  30 type ServerRequiredSuite struct{}
  31
  32 func pythonDir() string {
  33         gopath := os.Getenv("GOPATH")
  34         return fmt.Sprintf("%s/../../sdk/python", strings.Split(gopath, ":")[0])
  35 }
  36
  37 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  38         cwd, _ := os.Getwd()
  39         defer os.Chdir(cwd)
  40
  41         os.Chdir(pythonDir())
  42
  43         if err := exec.Command("python", "run_test_server.py", "start").Run(); err != nil {
  44                 panic("'python run_test_server.py start' returned error")
  45         }
  46         if err := exec.Command("python", "run_test_server.py", "start_keep").Run(); err != nil {
  47                 panic("'python run_test_server.py start_keep' returned error")
  48         }
  49
  50         os.Setenv("ARVADOS_API_HOST", "localhost:3001")
  51         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
  52         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
  53 }
  54
  55 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  56         cwd, _ := os.Getwd()
  57         defer os.Chdir(cwd)
  58
  59         os.Chdir(pythonDir())
  60         exec.Command("python", "run_test_server.py", "stop_keep").Run()
  61         exec.Command("python", "run_test_server.py", "stop").Run()
  62 }
  63
  64 func setupProxyService() {
  65
  66         client := &http.Client{Transport: &http.Transport{
  67                 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
  68
  69         var req *http.Request
  70         var err error
  71         if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
  72                 panic(err.Error())
  73         }
  74         req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
  75
  76         reader, writer := io.Pipe()
  77
  78         req.Body = reader
  79
  80         go func() {
  81                 data := url.Values{}
  82                 data.Set("keep_service", `{
  83   "service_host": "localhost",
  84   "service_port": 29950,
  85   "service_ssl_flag": false,
  86   "service_type": "proxy"
  87 }`)
  88
  89                 writer.Write([]byte(data.Encode()))
  90                 writer.Close()
  91         }()
  92
  93         var resp *http.Response
  94         if resp, err = client.Do(req); err != nil {
  95                 panic(err.Error())
  96         }
  97         if resp.StatusCode != 200 {
  98                 panic(resp.Status)
  99         }
 100 }
 101
 102 func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient {
 103         os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
 104         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 105
 106         go main()
 107         time.Sleep(100 * time.Millisecond)
 108
 109         os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
 110         os.Setenv("ARVADOS_API_TOKEN", token)
 111         kc, err := keepclient.MakeKeepClient()
 112         c.Check(kc.Using_proxy, Equals, true)
 113         c.Check(len(kc.ServiceRoots()), Equals, 1)
 114         c.Check(kc.ServiceRoots()[0], Equals, fmt.Sprintf("http://localhost:%v", port))
 115         c.Check(err, Equals, nil)
 116         os.Setenv("ARVADOS_KEEP_PROXY", "")
 117         log.Print("keepclient created")
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 122         log.Print("TestPutAndGet start")
 123
 124         os.Args = []string{"keepproxy", "-listen=:29950"}
 125         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 126         go main()
 127         time.Sleep(100 * time.Millisecond)
 128
 129         setupProxyService()
 130
 131         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
 132         kc, err := keepclient.MakeKeepClient()
 133         c.Check(kc.External, Equals, true)
 134         c.Check(kc.Using_proxy, Equals, true)
 135         c.Check(len(kc.ServiceRoots()), Equals, 1)
 136         c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:29950")
 137         c.Check(err, Equals, nil)
 138         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 139         log.Print("keepclient created")
 140
 141         defer listener.Close()
 142
 143         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 144         var hash2 string
 145
 146         {
 147                 _, _, err := kc.Ask(hash)
 148                 c.Check(err, Equals, keepclient.BlockNotFound)
 149                 log.Print("Ask 1")
 150         }
 151
 152         {
 153                 var rep int
 154                 var err error
 155                 hash2, rep, err = kc.PutB([]byte("foo"))
 156                 c.Check(hash2, Equals, fmt.Sprintf("%s+3", hash))
 157                 c.Check(rep, Equals, 2)
 158                 c.Check(err, Equals, nil)
 159                 log.Print("PutB")
 160         }
 161
 162         {
 163                 blocklen, _, err := kc.Ask(hash2)
 164                 c.Assert(err, Equals, nil)
 165                 c.Check(blocklen, Equals, int64(3))
 166                 log.Print("Ask 2")
 167         }
 168
 169         {
 170                 reader, blocklen, _, err := kc.Get(hash2)
 171                 c.Assert(err, Equals, nil)
 172                 all, err := ioutil.ReadAll(reader)
 173                 c.Check(all, DeepEquals, []byte("foo"))
 174                 c.Check(blocklen, Equals, int64(3))
 175                 log.Print("Get")
 176         }
 177
 178         log.Print("TestPutAndGet done")
 179 }
 180
 181 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 182         log.Print("TestPutAndGet start")
 183
 184         kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951)
 185         defer listener.Close()
 186
 187         log.Print("keepclient created")
 188
 189         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 190
 191         {
 192                 _, _, err := kc.Ask(hash)
 193                 c.Check(err, Equals, keepclient.BlockNotFound)
 194                 log.Print("Ask 1")
 195         }
 196
 197         {
 198                 hash2, rep, err := kc.PutB([]byte("bar"))
 199                 c.Check(hash2, Equals, "")
 200                 c.Check(rep, Equals, 0)
 201                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 202                 log.Print("PutB")
 203         }
 204
 205         {
 206                 blocklen, _, err := kc.Ask(hash)
 207                 c.Assert(err, Equals, keepclient.BlockNotFound)
 208                 c.Check(blocklen, Equals, int64(0))
 209                 log.Print("Ask 2")
 210         }
 211
 212         {
 213                 _, blocklen, _, err := kc.Get(hash)
 214                 c.Assert(err, Equals, keepclient.BlockNotFound)
 215                 c.Check(blocklen, Equals, int64(0))
 216                 log.Print("Get")
 217         }
 218
 219         log.Print("TestPutAndGetForbidden done")
 220 }
 221
 222 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 223         log.Print("TestGetDisabled start")
 224
 225         kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952)
 226         defer listener.Close()
 227
 228         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 229
 230         {
 231                 _, _, err := kc.Ask(hash)
 232                 c.Check(err, Equals, keepclient.BlockNotFound)
 233                 log.Print("Ask 1")
 234         }
 235
 236         {
 237                 hash2, rep, err := kc.PutB([]byte("baz"))
 238                 c.Check(hash2, Equals, fmt.Sprintf("%s+3", hash))
 239                 c.Check(rep, Equals, 2)
 240                 c.Check(err, Equals, nil)
 241                 log.Print("PutB")
 242         }
 243
 244         {
 245                 blocklen, _, err := kc.Ask(hash)
 246                 c.Assert(err, Equals, keepclient.BlockNotFound)
 247                 c.Check(blocklen, Equals, int64(0))
 248                 log.Print("Ask 2")
 249         }
 250
 251         {
 252                 _, blocklen, _, err := kc.Get(hash)
 253                 c.Assert(err, Equals, keepclient.BlockNotFound)
 254                 c.Check(blocklen, Equals, int64(0))
 255                 log.Print("Get")
 256         }
 257
 258         log.Print("TestGetDisabled done")
 259 }
 260
 261 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 262         log.Print("TestPutDisabled start")
 263
 264         kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953)
 265         defer listener.Close()
 266
 267         {
 268                 hash2, rep, err := kc.PutB([]byte("quux"))
 269                 c.Check(hash2, Equals, "")
 270                 c.Check(rep, Equals, 0)
 271                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 272                 log.Print("PutB")
 273         }
 274
 275         log.Print("TestPutDisabled done")
 276 }