services/keep-web/server_test.go

   1 package main
   2
   3 import (
   4         "crypto/md5"
   5         "fmt"
   6         "io"
   7         "io/ioutil"
   8         "net"
   9         "os/exec"
  10         "strings"
  11         "testing"
  12
  13         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  14         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  15         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  16         check "gopkg.in/check.v1"
  17 )
  18
  19 var _ = check.Suite(&IntegrationSuite{})
  20
  21 // IntegrationSuite tests need an API server and a keep-web server
  22 type IntegrationSuite struct {
  23         testServer *server
  24 }
  25
  26 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  27         for _, token := range []string{
  28                 "",
  29                 "bogustoken",
  30         } {
  31                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  32                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  33                 c.Check(body, check.Equals, "")
  34
  35                 if token != "" {
  36                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  37                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  38                         c.Check(body, check.Equals, "")
  39                 }
  40
  41                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  42                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  43                 c.Check(body, check.Equals, "")
  44         }
  45 }
  46
  47 // TODO: Move most cases to functional tests -- at least use Go's own
  48 // http client instead of forking curl. Just leave enough of an
  49 // integration test to assure that the documented way of invoking curl
  50 // really works against the server.
  51 func (s *IntegrationSuite) Test404(c *check.C) {
  52         for _, uri := range []string{
  53                 // Routing errors (always 404 regardless of what's stored in Keep)
  54                 "/",
  55                 "/foo",
  56                 "/download",
  57                 "/collections",
  58                 "/collections/",
  59                 // Implicit/generated index is not implemented yet;
  60                 // until then, return 404.
  61                 "/collections/" + arvadostest.FooCollection,
  62                 "/collections/" + arvadostest.FooCollection + "/",
  63                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  64                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  65                 // Non-existent file in collection
  66                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  67                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  68                 // Non-existent collection
  69                 "/collections/" + arvadostest.NonexistentCollection,
  70                 "/collections/" + arvadostest.NonexistentCollection + "/",
  71                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  72                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  73         } {
  74                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  75                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  76                 c.Check(body, check.Equals, "")
  77         }
  78 }
  79
  80 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  81         if testing.Short() {
  82                 c.Skip("skipping 1GB integration test in short mode")
  83         }
  84         s.test100BlockFile(c, 10000000)
  85 }
  86
  87 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
  88         if testing.Short() {
  89                 // 3 MB
  90                 s.test100BlockFile(c, 30000)
  91         } else {
  92                 // 300 MB
  93                 s.test100BlockFile(c, 3000000)
  94         }
  95 }
  96
  97 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
  98         testdata := make([]byte, blocksize)
  99         for i := 0; i < blocksize; i++ {
 100                 testdata[i] = byte(' ')
 101         }
 102         arv, err := arvadosclient.MakeArvadosClient()
 103         c.Assert(err, check.Equals, nil)
 104         arv.ApiToken = arvadostest.ActiveToken
 105         kc, err := keepclient.MakeKeepClient(&arv)
 106         c.Assert(err, check.Equals, nil)
 107         loc, _, err := kc.PutB(testdata[:])
 108         c.Assert(err, check.Equals, nil)
 109         mtext := "."
 110         for i := 0; i < 100; i++ {
 111                 mtext = mtext + " " + loc
 112         }
 113         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 114         coll := map[string]interface{}{}
 115         err = arv.Create("collections",
 116                 map[string]interface{}{
 117                         "collection": map[string]interface{}{
 118                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 119                                 "manifest_text": mtext,
 120                         },
 121                 }, &coll)
 122         c.Assert(err, check.Equals, nil)
 123         uuid := coll["uuid"].(string)
 124
 125         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 126         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 127         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 128         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 129         c.Check(size, check.Equals, int64(blocksize)*100)
 130 }
 131
 132 type curlCase struct {
 133         auth    string
 134         host    string
 135         path    string
 136         dataMD5 string
 137 }
 138
 139 func (s *IntegrationSuite) Test200(c *check.C) {
 140         anonymousTokens = []string{arvadostest.AnonymousToken}
 141         for _, spec := range []curlCase{
 142                 // My collection
 143                 {
 144                         auth:    arvadostest.ActiveToken,
 145                         host:    arvadostest.FooCollection + "--collections.example.com",
 146                         path:    "/foo",
 147                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 148                 },
 149                 {
 150                         auth:    arvadostest.ActiveToken,
 151                         host:    arvadostest.FooCollection + ".collections.example.com",
 152                         path:    "/foo",
 153                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 154                 },
 155                 {
 156                         host:    strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
 157                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 158                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 159                 },
 160                 {
 161                         path:    "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
 162                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 163                 },
 164                 {
 165                         path:    "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 166                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 167                 },
 168                 {
 169                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 170                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 171                 },
 172                 {
 173                         auth:    "tokensobogus",
 174                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 175                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 176                 },
 177                 {
 178                         auth:    arvadostest.ActiveToken,
 179                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 180                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 181                 },
 182                 {
 183                         auth:    arvadostest.AnonymousToken,
 184                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 185                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 186                 },
 187
 188                 // Anonymously accessible data
 189                 {
 190                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 191                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 192                 },
 193                 {
 194                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 195                         path:    "/Hello%20world.txt",
 196                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 197                 },
 198                 {
 199                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 200                         path:    "/_/Hello%20world.txt",
 201                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 202                 },
 203                 {
 204                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 205                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 206                 },
 207                 {
 208                         auth:    arvadostest.ActiveToken,
 209                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 210                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 211                 },
 212                 {
 213                         auth:    arvadostest.SpectatorToken,
 214                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 215                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 216                 },
 217                 {
 218                         auth:    arvadostest.SpectatorToken,
 219                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 220                         path:    "/Hello%20world.txt",
 221                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 222                 },
 223                 {
 224                         auth:    arvadostest.SpectatorToken,
 225                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 226                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 227                 },
 228         } {
 229                 host := spec.host
 230                 if host == "" {
 231                         host = "collections.example.com"
 232                 }
 233                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 234                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 235                 if strings.HasSuffix(spec.path, ".txt") {
 236                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 237                         // TODO: Check some types that aren't
 238                         // automatically detected by Go's http server
 239                         // by sniffing the content.
 240                 }
 241                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 242         }
 243 }
 244
 245 // Return header block and body.
 246 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 247         curlArgs := []string{"--silent", "--show-error", "--include"}
 248         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 249         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 250         if token != "" {
 251                 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
 252         }
 253         curlArgs = append(curlArgs, args...)
 254         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 255         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 256         cmd := exec.Command("curl", curlArgs...)
 257         stdout, err := cmd.StdoutPipe()
 258         c.Assert(err, check.Equals, nil)
 259         cmd.Stderr = cmd.Stdout
 260         go cmd.Start()
 261         buf := make([]byte, 2<<27)
 262         n, err := io.ReadFull(stdout, buf)
 263         // Discard (but measure size of) anything past 128 MiB.
 264         var discarded int64
 265         if err == io.ErrUnexpectedEOF {
 266                 err = nil
 267                 buf = buf[:n]
 268         } else {
 269                 c.Assert(err, check.Equals, nil)
 270                 discarded, err = io.Copy(ioutil.Discard, stdout)
 271                 c.Assert(err, check.Equals, nil)
 272         }
 273         err = cmd.Wait()
 274         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 275         // response from the server, even if the response status
 276         // indicates that the request failed. In our test suite, we
 277         // always expect a valid HTTP response, and we parse the
 278         // headers ourselves. If curl exits non-zero, our testing
 279         // environment is broken.
 280         c.Assert(err, check.Equals, nil)
 281         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 282         c.Assert(len(hdrsAndBody), check.Equals, 2)
 283         hdr = hdrsAndBody[0]
 284         bodyPart = hdrsAndBody[1]
 285         bodySize = int64(len(bodyPart)) + discarded
 286         return
 287 }
 288
 289 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 290         arvadostest.StartAPI()
 291         arvadostest.StartKeep(2, true)
 292
 293         arv, err := arvadosclient.MakeArvadosClient()
 294         c.Assert(err, check.Equals, nil)
 295         arv.ApiToken = arvadostest.ActiveToken
 296         kc, err := keepclient.MakeKeepClient(&arv)
 297         c.Assert(err, check.Equals, nil)
 298         kc.PutB([]byte("Hello world\n"))
 299         kc.PutB([]byte("foo"))
 300         kc.PutB([]byte("foobar"))
 301 }
 302
 303 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 304         arvadostest.StopKeep(2)
 305         arvadostest.StopAPI()
 306 }
 307
 308 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 309         arvadostest.ResetEnv()
 310         s.testServer = &server{}
 311         var err error
 312         address = "127.0.0.1:0"
 313         err = s.testServer.Start()
 314         c.Assert(err, check.Equals, nil)
 315 }
 316
 317 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 318         var err error
 319         if s.testServer != nil {
 320                 err = s.testServer.Close()
 321         }
 322         c.Check(err, check.Equals, nil)
 323 }
 324
 325 // Gocheck boilerplate
 326 func Test(t *testing.T) {
 327         check.TestingT(t)
 328 }