services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "crypto/md5"
   9         "fmt"
  10         "io"
  11         "io/ioutil"
  12         "net"
  13         "os"
  14         "os/exec"
  15         "strings"
  16         "testing"
  17
  18         "git.curoverse.com/arvados.git/sdk/go/arvados"
  19         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  20         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  21         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  22         check "gopkg.in/check.v1"
  23 )
  24
  25 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  26
  27 var _ = check.Suite(&IntegrationSuite{})
  28
  29 // IntegrationSuite tests need an API server and a keep-web server
  30 type IntegrationSuite struct {
  31         testServer *server
  32 }
  33
  34 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  35         for _, token := range []string{
  36                 "",
  37                 "bogustoken",
  38         } {
  39                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  40                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  41                 c.Check(body, check.Equals, "")
  42
  43                 if token != "" {
  44                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  45                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  46                         c.Check(body, check.Equals, "")
  47                 }
  48
  49                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  50                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  51                 c.Check(body, check.Equals, "")
  52         }
  53 }
  54
  55 // TODO: Move most cases to functional tests -- at least use Go's own
  56 // http client instead of forking curl. Just leave enough of an
  57 // integration test to assure that the documented way of invoking curl
  58 // really works against the server.
  59 func (s *IntegrationSuite) Test404(c *check.C) {
  60         for _, uri := range []string{
  61                 // Routing errors (always 404 regardless of what's stored in Keep)
  62                 "/",
  63                 "/foo",
  64                 "/download",
  65                 "/collections",
  66                 "/collections/",
  67                 // Implicit/generated index is not implemented yet;
  68                 // until then, return 404.
  69                 "/collections/" + arvadostest.FooCollection,
  70                 "/collections/" + arvadostest.FooCollection + "/",
  71                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  72                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  73                 // Non-existent file in collection
  74                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  75                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  76                 // Non-existent collection
  77                 "/collections/" + arvadostest.NonexistentCollection,
  78                 "/collections/" + arvadostest.NonexistentCollection + "/",
  79                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  80                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  81         } {
  82                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  83                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  84                 if len(body) > 0 {
  85                         c.Check(body, check.Equals, "404 page not found\n")
  86                 }
  87         }
  88 }
  89
  90 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  91         if testing.Short() {
  92                 c.Skip("skipping 1GB integration test in short mode")
  93         }
  94         s.test100BlockFile(c, 10000000)
  95 }
  96
  97 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
  98         if testing.Short() {
  99                 // 3 MB
 100                 s.test100BlockFile(c, 30000)
 101         } else {
 102                 // 300 MB
 103                 s.test100BlockFile(c, 3000000)
 104         }
 105 }
 106
 107 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 108         testdata := make([]byte, blocksize)
 109         for i := 0; i < blocksize; i++ {
 110                 testdata[i] = byte(' ')
 111         }
 112         arv, err := arvadosclient.MakeArvadosClient()
 113         c.Assert(err, check.Equals, nil)
 114         arv.ApiToken = arvadostest.ActiveToken
 115         kc, err := keepclient.MakeKeepClient(arv)
 116         c.Assert(err, check.Equals, nil)
 117         loc, _, err := kc.PutB(testdata[:])
 118         c.Assert(err, check.Equals, nil)
 119         mtext := "."
 120         for i := 0; i < 100; i++ {
 121                 mtext = mtext + " " + loc
 122         }
 123         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 124         coll := map[string]interface{}{}
 125         err = arv.Create("collections",
 126                 map[string]interface{}{
 127                         "collection": map[string]interface{}{
 128                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 129                                 "manifest_text": mtext,
 130                         },
 131                 }, &coll)
 132         c.Assert(err, check.Equals, nil)
 133         uuid := coll["uuid"].(string)
 134
 135         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 136         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 137         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 138         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 139         c.Check(size, check.Equals, int64(blocksize)*100)
 140 }
 141
 142 type curlCase struct {
 143         auth    string
 144         host    string
 145         path    string
 146         dataMD5 string
 147 }
 148
 149 func (s *IntegrationSuite) Test200(c *check.C) {
 150         s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken}
 151         for _, spec := range []curlCase{
 152                 // My collection
 153                 {
 154                         auth:    arvadostest.ActiveToken,
 155                         host:    arvadostest.FooCollection + "--collections.example.com",
 156                         path:    "/foo",
 157                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 158                 },
 159                 {
 160                         auth:    arvadostest.ActiveToken,
 161                         host:    arvadostest.FooCollection + ".collections.example.com",
 162                         path:    "/foo",
 163                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 164                 },
 165                 {
 166                         host:    strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com",
 167                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 168                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 169                 },
 170                 {
 171                         path:    "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo",
 172                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 173                 },
 174                 {
 175                         path:    "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 176                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 177                 },
 178                 {
 179                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 180                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 181                 },
 182                 {
 183                         auth:    "tokensobogus",
 184                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 185                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 186                 },
 187                 {
 188                         auth:    arvadostest.ActiveToken,
 189                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 190                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 191                 },
 192                 {
 193                         auth:    arvadostest.AnonymousToken,
 194                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 195                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 196                 },
 197
 198                 // Anonymously accessible data
 199                 {
 200                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 201                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 202                 },
 203                 {
 204                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 205                         path:    "/Hello%20world.txt",
 206                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 207                 },
 208                 {
 209                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 210                         path:    "/_/Hello%20world.txt",
 211                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 212                 },
 213                 {
 214                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 215                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 216                 },
 217                 {
 218                         auth:    arvadostest.ActiveToken,
 219                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 220                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 221                 },
 222                 {
 223                         auth:    arvadostest.SpectatorToken,
 224                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 225                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 226                 },
 227                 {
 228                         auth:    arvadostest.SpectatorToken,
 229                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 230                         path:    "/Hello%20world.txt",
 231                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 232                 },
 233                 {
 234                         auth:    arvadostest.SpectatorToken,
 235                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 236                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 237                 },
 238         } {
 239                 host := spec.host
 240                 if host == "" {
 241                         host = "collections.example.com"
 242                 }
 243                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 244                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 245                 if strings.HasSuffix(spec.path, ".txt") {
 246                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 247                         // TODO: Check some types that aren't
 248                         // automatically detected by Go's http server
 249                         // by sniffing the content.
 250                 }
 251                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 252         }
 253 }
 254
 255 // Return header block and body.
 256 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 257         curlArgs := []string{"--silent", "--show-error", "--include"}
 258         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 259         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 260         if token != "" {
 261                 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
 262         }
 263         curlArgs = append(curlArgs, args...)
 264         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 265         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 266         cmd := exec.Command("curl", curlArgs...)
 267         stdout, err := cmd.StdoutPipe()
 268         c.Assert(err, check.Equals, nil)
 269         cmd.Stderr = cmd.Stdout
 270         go cmd.Start()
 271         buf := make([]byte, 2<<27)
 272         n, err := io.ReadFull(stdout, buf)
 273         // Discard (but measure size of) anything past 128 MiB.
 274         var discarded int64
 275         if err == io.ErrUnexpectedEOF {
 276                 err = nil
 277                 buf = buf[:n]
 278         } else {
 279                 c.Assert(err, check.Equals, nil)
 280                 discarded, err = io.Copy(ioutil.Discard, stdout)
 281                 c.Assert(err, check.Equals, nil)
 282         }
 283         err = cmd.Wait()
 284         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 285         // response from the server, even if the response status
 286         // indicates that the request failed. In our test suite, we
 287         // always expect a valid HTTP response, and we parse the
 288         // headers ourselves. If curl exits non-zero, our testing
 289         // environment is broken.
 290         c.Assert(err, check.Equals, nil)
 291         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 292         c.Assert(len(hdrsAndBody), check.Equals, 2)
 293         hdr = hdrsAndBody[0]
 294         bodyPart = hdrsAndBody[1]
 295         bodySize = int64(len(bodyPart)) + discarded
 296         return
 297 }
 298
 299 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 300         arvadostest.StartAPI()
 301         arvadostest.StartKeep(2, true)
 302
 303         arv, err := arvadosclient.MakeArvadosClient()
 304         c.Assert(err, check.Equals, nil)
 305         arv.ApiToken = arvadostest.ActiveToken
 306         kc, err := keepclient.MakeKeepClient(arv)
 307         c.Assert(err, check.Equals, nil)
 308         kc.PutB([]byte("Hello world\n"))
 309         kc.PutB([]byte("foo"))
 310         kc.PutB([]byte("foobar"))
 311 }
 312
 313 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 314         arvadostest.StopKeep(2)
 315         arvadostest.StopAPI()
 316 }
 317
 318 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 319         arvadostest.ResetEnv()
 320         cfg := DefaultConfig()
 321         cfg.Client = arvados.Client{
 322                 APIHost:  testAPIHost,
 323                 Insecure: true,
 324         }
 325         cfg.Listen = "127.0.0.1:0"
 326         s.testServer = &server{Config: cfg}
 327         err := s.testServer.Start()
 328         c.Assert(err, check.Equals, nil)
 329 }
 330
 331 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 332         var err error
 333         if s.testServer != nil {
 334                 err = s.testServer.Close()
 335         }
 336         c.Check(err, check.Equals, nil)
 337 }
 338
 339 // Gocheck boilerplate
 340 func Test(t *testing.T) {
 341         check.TestingT(t)
 342 }