services/keep-web/server_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "fmt"
   6         "io"
   7         "io/ioutil"
   8         "net"
   9         "os"
  10         "os/exec"
  11         "strings"
  12         "testing"
  13
  14         "git.curoverse.com/arvados.git/sdk/go/arvados"
  15         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  16         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  17         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  18         check "gopkg.in/check.v1"
  19 )
  20
  21 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  22
  23 var _ = check.Suite(&IntegrationSuite{})
  24
  25 // IntegrationSuite tests need an API server and a keep-web server
  26 type IntegrationSuite struct {
  27         testServer *server
  28 }
  29
  30 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  31         for _, token := range []string{
  32                 "",
  33                 "bogustoken",
  34         } {
  35                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  36                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  37                 c.Check(body, check.Equals, "")
  38
  39                 if token != "" {
  40                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  41                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  42                         c.Check(body, check.Equals, "")
  43                 }
  44
  45                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  46                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  47                 c.Check(body, check.Equals, "")
  48         }
  49 }
  50
  51 // TODO: Move most cases to functional tests -- at least use Go's own
  52 // http client instead of forking curl. Just leave enough of an
  53 // integration test to assure that the documented way of invoking curl
  54 // really works against the server.
  55 func (s *IntegrationSuite) Test404(c *check.C) {
  56         for _, uri := range []string{
  57                 // Routing errors (always 404 regardless of what's stored in Keep)
  58                 "/",
  59                 "/foo",
  60                 "/download",
  61                 "/collections",
  62                 "/collections/",
  63                 // Implicit/generated index is not implemented yet;
  64                 // until then, return 404.
  65                 "/collections/" + arvadostest.FooCollection,
  66                 "/collections/" + arvadostest.FooCollection + "/",
  67                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  68                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  69                 // Non-existent file in collection
  70                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  71                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  72                 // Non-existent collection
  73                 "/collections/" + arvadostest.NonexistentCollection,
  74                 "/collections/" + arvadostest.NonexistentCollection + "/",
  75                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  76                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  77         } {
  78                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  79                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  80                 if len(body) > 0 {
  81                         c.Check(body, check.Equals, "404 page not found\n")
  82                 }
  83         }
  84 }
  85
  86 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  87         if testing.Short() {
  88                 c.Skip("skipping 1GB integration test in short mode")
  89         }
  90         s.test100BlockFile(c, 10000000)
  91 }
  92
  93 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
  94         if testing.Short() {
  95                 // 3 MB
  96                 s.test100BlockFile(c, 30000)
  97         } else {
  98                 // 300 MB
  99                 s.test100BlockFile(c, 3000000)
 100         }
 101 }
 102
 103 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 104         testdata := make([]byte, blocksize)
 105         for i := 0; i < blocksize; i++ {
 106                 testdata[i] = byte(' ')
 107         }
 108         arv, err := arvadosclient.MakeArvadosClient()
 109         c.Assert(err, check.Equals, nil)
 110         arv.ApiToken = arvadostest.ActiveToken
 111         kc, err := keepclient.MakeKeepClient(arv)
 112         c.Assert(err, check.Equals, nil)
 113         loc, _, err := kc.PutB(testdata[:])
 114         c.Assert(err, check.Equals, nil)
 115         mtext := "."
 116         for i := 0; i < 100; i++ {
 117                 mtext = mtext + " " + loc
 118         }
 119         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 120         coll := map[string]interface{}{}
 121         err = arv.Create("collections",
 122                 map[string]interface{}{
 123                         "collection": map[string]interface{}{
 124                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 125                                 "manifest_text": mtext,
 126                         },
 127                 }, &coll)
 128         c.Assert(err, check.Equals, nil)
 129         uuid := coll["uuid"].(string)
 130
 131         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 132         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 133         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 134         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 135         c.Check(size, check.Equals, int64(blocksize)*100)
 136 }
 137
 138 type curlCase struct {
 139         auth    string
 140         host    string
 141         path    string
 142         dataMD5 string
 143 }
 144
 145 func (s *IntegrationSuite) Test200(c *check.C) {
 146         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 147         for _, spec := range []curlCase{
 148                 // My collection
 149                 {
 150                         auth:    arvadostest.ActiveToken,
 151                         host:    arvadostest.FooCollection + "--collections.example.com",
 152                         path:    "/foo",
 153                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 154                 },
 155                 {
 156                         auth:    arvadostest.ActiveToken,
 157                         host:    arvadostest.FooCollection + ".collections.example.com",
 158                         path:    "/foo",
 159                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 160                 },
 161                 {
 162                         host:    strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
 163                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 164                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 165                 },
 166                 {
 167                         path:    "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
 168                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 169                 },
 170                 {
 171                         path:    "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 172                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 173                 },
 174                 {
 175                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 176                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 177                 },
 178                 {
 179                         auth:    "tokensobogus",
 180                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 181                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 182                 },
 183                 {
 184                         auth:    arvadostest.ActiveToken,
 185                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 186                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 187                 },
 188                 {
 189                         auth:    arvadostest.AnonymousToken,
 190                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 191                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 192                 },
 193
 194                 // Anonymously accessible data
 195                 {
 196                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 197                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 198                 },
 199                 {
 200                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 201                         path:    "/Hello%20world.txt",
 202                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 203                 },
 204                 {
 205                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 206                         path:    "/_/Hello%20world.txt",
 207                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 208                 },
 209                 {
 210                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 211                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 212                 },
 213                 {
 214                         auth:    arvadostest.ActiveToken,
 215                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 216                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 217                 },
 218                 {
 219                         auth:    arvadostest.SpectatorToken,
 220                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 221                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 222                 },
 223                 {
 224                         auth:    arvadostest.SpectatorToken,
 225                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 226                         path:    "/Hello%20world.txt",
 227                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 228                 },
 229                 {
 230                         auth:    arvadostest.SpectatorToken,
 231                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 232                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 233                 },
 234         } {
 235                 host := spec.host
 236                 if host == "" {
 237                         host = "collections.example.com"
 238                 }
 239                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 240                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 241                 if strings.HasSuffix(spec.path, ".txt") {
 242                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 243                         // TODO: Check some types that aren't
 244                         // automatically detected by Go's http server
 245                         // by sniffing the content.
 246                 }
 247                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 248         }
 249 }
 250
 251 // Return header block and body.
 252 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 253         curlArgs := []string{"--silent", "--show-error", "--include"}
 254         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 255         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 256         if token != "" {
 257                 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
 258         }
 259         curlArgs = append(curlArgs, args...)
 260         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 261         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 262         cmd := exec.Command("curl", curlArgs...)
 263         stdout, err := cmd.StdoutPipe()
 264         c.Assert(err, check.Equals, nil)
 265         cmd.Stderr = cmd.Stdout
 266         go cmd.Start()
 267         buf := make([]byte, 2<<27)
 268         n, err := io.ReadFull(stdout, buf)
 269         // Discard (but measure size of) anything past 128 MiB.
 270         var discarded int64
 271         if err == io.ErrUnexpectedEOF {
 272                 err = nil
 273                 buf = buf[:n]
 274         } else {
 275                 c.Assert(err, check.Equals, nil)
 276                 discarded, err = io.Copy(ioutil.Discard, stdout)
 277                 c.Assert(err, check.Equals, nil)
 278         }
 279         err = cmd.Wait()
 280         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 281         // response from the server, even if the response status
 282         // indicates that the request failed. In our test suite, we
 283         // always expect a valid HTTP response, and we parse the
 284         // headers ourselves. If curl exits non-zero, our testing
 285         // environment is broken.
 286         c.Assert(err, check.Equals, nil)
 287         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 288         c.Assert(len(hdrsAndBody), check.Equals, 2)
 289         hdr = hdrsAndBody[0]
 290         bodyPart = hdrsAndBody[1]
 291         bodySize = int64(len(bodyPart)) + discarded
 292         return
 293 }
 294
 295 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 296         arvadostest.StartAPI()
 297         arvadostest.StartKeep(2, true)
 298
 299         arv, err := arvadosclient.MakeArvadosClient()
 300         c.Assert(err, check.Equals, nil)
 301         arv.ApiToken = arvadostest.ActiveToken
 302         kc, err := keepclient.MakeKeepClient(arv)
 303         c.Assert(err, check.Equals, nil)
 304         kc.PutB([]byte("Hello world\n"))
 305         kc.PutB([]byte("foo"))
 306         kc.PutB([]byte("foobar"))
 307 }
 308
 309 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 310         arvadostest.StopKeep(2)
 311         arvadostest.StopAPI()
 312 }
 313
 314 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 315         arvadostest.ResetEnv()
 316         cfg := DefaultConfig()
 317         cfg.Client = arvados.Client{
 318                 APIHost:  testAPIHost,
 319                 Insecure: true,
 320         }
 321         cfg.Listen = "127.0.0.1:0"
 322         s.testServer = &server{Config: cfg}
 323         err := s.testServer.Start()
 324         c.Assert(err, check.Equals, nil)
 325 }
 326
 327 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 328         var err error
 329         if s.testServer != nil {
 330                 err = s.testServer.Close()
 331         }
 332         c.Check(err, check.Equals, nil)
 333 }
 334
 335 // Gocheck boilerplate
 336 func Test(t *testing.T) {
 337         check.TestingT(t)
 338 }