services/keep/src/arvados.org/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "arvados.org/keepclient"
   5         "crypto/md5"
   6         "crypto/tls"
   7         "fmt"
   8         . "gopkg.in/check.v1"
   9         "io"
  10         "io/ioutil"
  11         "log"
  12         "net/http"
  13         "net/url"
  14         "os"
  15         "os/exec"
  16         "strings"
  17         "testing"
  18         "time"
  19 )
  20
  21 // Gocheck boilerplate
  22 func Test(t *testing.T) {
  23         TestingT(t)
  24 }
  25
  26 // Gocheck boilerplate
  27 var _ = Suite(&ServerRequiredSuite{})
  28
  29 // Tests that require the Keep server running
  30 type ServerRequiredSuite struct{}
  31
  32 func pythonDir() string {
  33         gopath := os.Getenv("GOPATH")
  34         return fmt.Sprintf("%s/../../sdk/python", strings.Split(gopath, ":")[0])
  35 }
  36
  37 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  38         cwd, _ := os.Getwd()
  39         defer os.Chdir(cwd)
  40
  41         os.Chdir(pythonDir())
  42
  43         if err := exec.Command("python", "run_test_server.py", "start").Run(); err != nil {
  44                 panic("'python run_test_server.py start' returned error")
  45         }
  46         if err := exec.Command("python", "run_test_server.py", "start_keep").Run(); err != nil {
  47                 panic("'python run_test_server.py start_keep' returned error")
  48         }
  49
  50         os.Setenv("ARVADOS_API_HOST", "localhost:3001")
  51         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
  52         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
  53 }
  54
  55 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  56         cwd, _ := os.Getwd()
  57         defer os.Chdir(cwd)
  58
  59         os.Chdir(pythonDir())
  60         exec.Command("python", "run_test_server.py", "stop_keep").Run()
  61         exec.Command("python", "run_test_server.py", "stop").Run()
  62 }
  63
  64 func setupProxyService() {
  65
  66         client := &http.Client{Transport: &http.Transport{
  67                 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
  68
  69         var req *http.Request
  70         var err error
  71         if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
  72                 panic(err.Error())
  73         }
  74         req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
  75
  76         reader, writer := io.Pipe()
  77
  78         req.Body = reader
  79
  80         go func() {
  81                 data := url.Values{}
  82                 data.Set("keep_service", `{
  83   "service_host": "localhost",
  84   "service_port": 29950,
  85   "service_ssl_flag": false,
  86   "service_type": "proxy"
  87 }`)
  88
  89                 writer.Write([]byte(data.Encode()))
  90                 writer.Close()
  91         }()
  92
  93         var resp *http.Response
  94         if resp, err = client.Do(req); err != nil {
  95                 panic(err.Error())
  96         }
  97         if resp.StatusCode != 200 {
  98                 panic(resp.Status)
  99         }
 100 }
 101
 102 func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient {
 103         os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
 104         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 105
 106         go main()
 107         time.Sleep(100 * time.Millisecond)
 108
 109         os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
 110         os.Setenv("ARVADOS_API_TOKEN", token)
 111         kc, err := keepclient.MakeKeepClient()
 112         c.Check(kc.Using_proxy, Equals, true)
 113         c.Check(len(kc.ServiceRoots()), Equals, 1)
 114         c.Check(kc.ServiceRoots()[0], Equals, fmt.Sprintf("http://localhost:%v", port))
 115         c.Check(err, Equals, nil)
 116         os.Setenv("ARVADOS_KEEP_PROXY", "")
 117         log.Print("keepclient created")
 118         return kc
 119 }
 120
 121 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 122         log.Print("TestPutAndGet start")
 123
 124         os.Args = []string{"keepproxy", "-listen=:29950"}
 125         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 126         go main()
 127         time.Sleep(100 * time.Millisecond)
 128
 129         setupProxyService()
 130
 131         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
 132         kc, err := keepclient.MakeKeepClient()
 133         c.Check(kc.External, Equals, true)
 134         c.Check(kc.Using_proxy, Equals, true)
 135         c.Check(len(kc.ServiceRoots()), Equals, 1)
 136         c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:29950")
 137         c.Check(err, Equals, nil)
 138         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 139         log.Print("keepclient created")
 140
 141         defer listener.Close()
 142
 143         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 144
 145         {
 146                 _, _, err := kc.Ask(hash)
 147                 c.Check(err, Equals, keepclient.BlockNotFound)
 148                 log.Print("Ask 1")
 149         }
 150
 151         {
 152                 hash2, rep, err := kc.PutB([]byte("foo"))
 153                 c.Check(hash2, Equals, hash)
 154                 c.Check(rep, Equals, 2)
 155                 c.Check(err, Equals, nil)
 156                 log.Print("PutB")
 157         }
 158
 159         {
 160                 blocklen, _, err := kc.Ask(hash)
 161                 c.Assert(err, Equals, nil)
 162                 c.Check(blocklen, Equals, int64(3))
 163                 log.Print("Ask 2")
 164         }
 165
 166         {
 167                 reader, blocklen, _, err := kc.Get(hash)
 168                 c.Assert(err, Equals, nil)
 169                 all, err := ioutil.ReadAll(reader)
 170                 c.Check(all, DeepEquals, []byte("foo"))
 171                 c.Check(blocklen, Equals, int64(3))
 172                 log.Print("Get")
 173         }
 174
 175         log.Print("TestPutAndGet done")
 176 }
 177
 178 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 179         log.Print("TestPutAndGet start")
 180
 181         kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951)
 182         defer listener.Close()
 183
 184         log.Print("keepclient created")
 185
 186         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 187
 188         {
 189                 _, _, err := kc.Ask(hash)
 190                 c.Check(err, Equals, keepclient.BlockNotFound)
 191                 log.Print("Ask 1")
 192         }
 193
 194         {
 195                 hash2, rep, err := kc.PutB([]byte("bar"))
 196                 c.Check(hash2, Equals, hash)
 197                 c.Check(rep, Equals, 0)
 198                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 199                 log.Print("PutB")
 200         }
 201
 202         {
 203                 blocklen, _, err := kc.Ask(hash)
 204                 c.Assert(err, Equals, keepclient.BlockNotFound)
 205                 c.Check(blocklen, Equals, int64(0))
 206                 log.Print("Ask 2")
 207         }
 208
 209         {
 210                 _, blocklen, _, err := kc.Get(hash)
 211                 c.Assert(err, Equals, keepclient.BlockNotFound)
 212                 c.Check(blocklen, Equals, int64(0))
 213                 log.Print("Get")
 214         }
 215
 216         log.Print("TestPutAndGetForbidden done")
 217 }
 218
 219 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 220         log.Print("TestGetDisabled start")
 221
 222         kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952)
 223         defer listener.Close()
 224
 225         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 226
 227         {
 228                 _, _, err := kc.Ask(hash)
 229                 c.Check(err, Equals, keepclient.BlockNotFound)
 230                 log.Print("Ask 1")
 231         }
 232
 233         {
 234                 hash2, rep, err := kc.PutB([]byte("baz"))
 235                 c.Check(hash2, Equals, hash)
 236                 c.Check(rep, Equals, 2)
 237                 c.Check(err, Equals, nil)
 238                 log.Print("PutB")
 239         }
 240
 241         {
 242                 blocklen, _, err := kc.Ask(hash)
 243                 c.Assert(err, Equals, keepclient.BlockNotFound)
 244                 c.Check(blocklen, Equals, int64(0))
 245                 log.Print("Ask 2")
 246         }
 247
 248         {
 249                 _, blocklen, _, err := kc.Get(hash)
 250                 c.Assert(err, Equals, keepclient.BlockNotFound)
 251                 c.Check(blocklen, Equals, int64(0))
 252                 log.Print("Get")
 253         }
 254
 255         log.Print("TestGetDisabled done")
 256 }
 257
 258 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 259         log.Print("TestPutDisabled start")
 260
 261         kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953)
 262         defer listener.Close()
 263
 264         hash := fmt.Sprintf("%x", md5.Sum([]byte("quux")))
 265
 266         {
 267                 hash2, rep, err := kc.PutB([]byte("quux"))
 268                 c.Check(hash2, Equals, hash)
 269                 c.Check(rep, Equals, 0)
 270                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 271                 log.Print("PutB")
 272         }
 273
 274         log.Print("TestPutDisabled done")
 275 }