services/keep/src/arvados.org/keepproxy/keepproxy_test.go

   1 package main
   2
   3 import (
   4         "arvados.org/keepclient"
   5         "arvados.org/sdk"
   6         "crypto/md5"
   7         "crypto/tls"
   8         "fmt"
   9         . "gopkg.in/check.v1"
  10         "io"
  11         "io/ioutil"
  12         "log"
  13         "net/http"
  14         "net/url"
  15         "os"
  16         "os/exec"
  17         "strings"
  18         "testing"
  19         "time"
  20 )
  21
  22 // Gocheck boilerplate
  23 func Test(t *testing.T) {
  24         TestingT(t)
  25 }
  26
  27 // Gocheck boilerplate
  28 var _ = Suite(&ServerRequiredSuite{})
  29
  30 // Tests that require the Keep server running
  31 type ServerRequiredSuite struct{}
  32
  33 func pythonDir() string {
  34         gopath := os.Getenv("GOPATH")
  35         return fmt.Sprintf("%s/../../sdk/python/tests", strings.Split(gopath, ":")[0])
  36 }
  37
  38 func (s *ServerRequiredSuite) SetUpSuite(c *C) {
  39         cwd, _ := os.Getwd()
  40         defer os.Chdir(cwd)
  41
  42         os.Chdir(pythonDir())
  43
  44         if err := exec.Command("python", "run_test_server.py", "start").Run(); err != nil {
  45                 panic("'python run_test_server.py start' returned error")
  46         }
  47         if err := exec.Command("python", "run_test_server.py", "start_keep").Run(); err != nil {
  48                 panic("'python run_test_server.py start_keep' returned error")
  49         }
  50
  51         os.Setenv("ARVADOS_API_HOST", "localhost:3001")
  52         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
  53         os.Setenv("ARVADOS_API_HOST_INSECURE", "true")
  54 }
  55
  56 func (s *ServerRequiredSuite) TearDownSuite(c *C) {
  57         cwd, _ := os.Getwd()
  58         defer os.Chdir(cwd)
  59
  60         os.Chdir(pythonDir())
  61         exec.Command("python", "run_test_server.py", "stop_keep").Run()
  62         exec.Command("python", "run_test_server.py", "stop").Run()
  63 }
  64
  65 func setupProxyService() {
  66
  67         client := &http.Client{Transport: &http.Transport{
  68                 TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
  69
  70         var req *http.Request
  71         var err error
  72         if req, err = http.NewRequest("POST", fmt.Sprintf("https://%s/arvados/v1/keep_services", os.Getenv("ARVADOS_API_HOST")), nil); err != nil {
  73                 panic(err.Error())
  74         }
  75         req.Header.Add("Authorization", fmt.Sprintf("OAuth2 %s", os.Getenv("ARVADOS_API_TOKEN")))
  76
  77         reader, writer := io.Pipe()
  78
  79         req.Body = reader
  80
  81         go func() {
  82                 data := url.Values{}
  83                 data.Set("keep_service", `{
  84   "service_host": "localhost",
  85   "service_port": 29950,
  86   "service_ssl_flag": false,
  87   "service_type": "proxy"
  88 }`)
  89
  90                 writer.Write([]byte(data.Encode()))
  91                 writer.Close()
  92         }()
  93
  94         var resp *http.Response
  95         if resp, err = client.Do(req); err != nil {
  96                 panic(err.Error())
  97         }
  98         if resp.StatusCode != 200 {
  99                 panic(resp.Status)
 100         }
 101 }
 102
 103 func runProxy(c *C, args []string, token string, port int) keepclient.KeepClient {
 104         os.Args = append(args, fmt.Sprintf("-listen=:%v", port))
 105         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 106
 107         go main()
 108         time.Sleep(100 * time.Millisecond)
 109
 110         os.Setenv("ARVADOS_KEEP_PROXY", fmt.Sprintf("http://localhost:%v", port))
 111         os.Setenv("ARVADOS_API_TOKEN", token)
 112         arv, err := sdk.MakeArvadosClient()
 113         kc, err := keepclient.MakeKeepClient(&arv)
 114         c.Check(kc.Using_proxy, Equals, true)
 115         c.Check(len(kc.ServiceRoots()), Equals, 1)
 116         c.Check(kc.ServiceRoots()[0], Equals, fmt.Sprintf("http://localhost:%v", port))
 117         c.Check(err, Equals, nil)
 118         os.Setenv("ARVADOS_KEEP_PROXY", "")
 119         log.Print("keepclient created")
 120         return kc
 121 }
 122
 123 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 124         log.Print("TestPutAndGet start")
 125
 126         os.Args = []string{"keepproxy", "-listen=:29950"}
 127         os.Setenv("ARVADOS_API_TOKEN", "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h")
 128         go main()
 129         time.Sleep(100 * time.Millisecond)
 130
 131         setupProxyService()
 132
 133         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "true")
 134         arv, err := sdk.MakeArvadosClient()
 135         kc, err := keepclient.MakeKeepClient(&arv)
 136         c.Check(kc.Arvados.External, Equals, true)
 137         c.Check(kc.Using_proxy, Equals, true)
 138         c.Check(len(kc.ServiceRoots()), Equals, 1)
 139         c.Check(kc.ServiceRoots()[0], Equals, "http://localhost:29950")
 140         c.Check(err, Equals, nil)
 141         os.Setenv("ARVADOS_EXTERNAL_CLIENT", "")
 142         log.Print("keepclient created")
 143
 144         defer listener.Close()
 145
 146         hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
 147         var hash2 string
 148
 149         {
 150                 _, _, err := kc.Ask(hash)
 151                 c.Check(err, Equals, keepclient.BlockNotFound)
 152                 log.Print("Ask 1")
 153         }
 154
 155         {
 156                 var rep int
 157                 var err error
 158                 hash2, rep, err = kc.PutB([]byte("foo"))
 159                 c.Check(hash2, Equals, fmt.Sprintf("%s+3", hash))
 160                 c.Check(rep, Equals, 2)
 161                 c.Check(err, Equals, nil)
 162                 log.Print("PutB")
 163         }
 164
 165         {
 166                 blocklen, _, err := kc.Ask(hash2)
 167                 c.Assert(err, Equals, nil)
 168                 c.Check(blocklen, Equals, int64(3))
 169                 log.Print("Ask 2")
 170         }
 171
 172         {
 173                 reader, blocklen, _, err := kc.Get(hash2)
 174                 c.Assert(err, Equals, nil)
 175                 all, err := ioutil.ReadAll(reader)
 176                 c.Check(all, DeepEquals, []byte("foo"))
 177                 c.Check(blocklen, Equals, int64(3))
 178                 log.Print("Get")
 179         }
 180
 181         log.Print("TestPutAndGet done")
 182 }
 183
 184 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 185         log.Print("TestPutAndGet start")
 186
 187         kc := runProxy(c, []string{"keepproxy"}, "123abc", 29951)
 188         defer listener.Close()
 189
 190         log.Print("keepclient created")
 191
 192         hash := fmt.Sprintf("%x", md5.Sum([]byte("bar")))
 193
 194         {
 195                 _, _, err := kc.Ask(hash)
 196                 c.Check(err, Equals, keepclient.BlockNotFound)
 197                 log.Print("Ask 1")
 198         }
 199
 200         {
 201                 hash2, rep, err := kc.PutB([]byte("bar"))
 202                 c.Check(hash2, Equals, "")
 203                 c.Check(rep, Equals, 0)
 204                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 205                 log.Print("PutB")
 206         }
 207
 208         {
 209                 blocklen, _, err := kc.Ask(hash)
 210                 c.Assert(err, Equals, keepclient.BlockNotFound)
 211                 c.Check(blocklen, Equals, int64(0))
 212                 log.Print("Ask 2")
 213         }
 214
 215         {
 216                 _, blocklen, _, err := kc.Get(hash)
 217                 c.Assert(err, Equals, keepclient.BlockNotFound)
 218                 c.Check(blocklen, Equals, int64(0))
 219                 log.Print("Get")
 220         }
 221
 222         log.Print("TestPutAndGetForbidden done")
 223 }
 224
 225 func (s *ServerRequiredSuite) TestGetDisabled(c *C) {
 226         log.Print("TestGetDisabled start")
 227
 228         kc := runProxy(c, []string{"keepproxy", "-no-get"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29952)
 229         defer listener.Close()
 230
 231         hash := fmt.Sprintf("%x", md5.Sum([]byte("baz")))
 232
 233         {
 234                 _, _, err := kc.Ask(hash)
 235                 c.Check(err, Equals, keepclient.BlockNotFound)
 236                 log.Print("Ask 1")
 237         }
 238
 239         {
 240                 hash2, rep, err := kc.PutB([]byte("baz"))
 241                 c.Check(hash2, Equals, fmt.Sprintf("%s+3", hash))
 242                 c.Check(rep, Equals, 2)
 243                 c.Check(err, Equals, nil)
 244                 log.Print("PutB")
 245         }
 246
 247         {
 248                 blocklen, _, err := kc.Ask(hash)
 249                 c.Assert(err, Equals, keepclient.BlockNotFound)
 250                 c.Check(blocklen, Equals, int64(0))
 251                 log.Print("Ask 2")
 252         }
 253
 254         {
 255                 _, blocklen, _, err := kc.Get(hash)
 256                 c.Assert(err, Equals, keepclient.BlockNotFound)
 257                 c.Check(blocklen, Equals, int64(0))
 258                 log.Print("Get")
 259         }
 260
 261         log.Print("TestGetDisabled done")
 262 }
 263
 264 func (s *ServerRequiredSuite) TestPutDisabled(c *C) {
 265         log.Print("TestPutDisabled start")
 266
 267         kc := runProxy(c, []string{"keepproxy", "-no-put"}, "4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h", 29953)
 268         defer listener.Close()
 269
 270         {
 271                 hash2, rep, err := kc.PutB([]byte("quux"))
 272                 c.Check(hash2, Equals, "")
 273                 c.Check(rep, Equals, 0)
 274                 c.Check(err, Equals, keepclient.InsufficientReplicasError)
 275                 log.Print("PutB")
 276         }
 277
 278         log.Print("TestPutDisabled done")
 279 }