services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "crypto/md5"
   9         "encoding/json"
  10         "fmt"
  11         "io"
  12         "io/ioutil"
  13         "net"
  14         "net/http"
  15         "os"
  16         "os/exec"
  17         "strings"
  18         "testing"
  19
  20         "git.curoverse.com/arvados.git/lib/config"
  21         "git.curoverse.com/arvados.git/sdk/go/arvados"
  22         "git.curoverse.com/arvados.git/sdk/go/arvadosclient"
  23         "git.curoverse.com/arvados.git/sdk/go/arvadostest"
  24         "git.curoverse.com/arvados.git/sdk/go/keepclient"
  25         check "gopkg.in/check.v1"
  26 )
  27
  28 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  29
  30 var _ = check.Suite(&IntegrationSuite{})
  31
  32 // IntegrationSuite tests need an API server and a keep-web server
  33 type IntegrationSuite struct {
  34         testServer *server
  35 }
  36
  37 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  38         for _, token := range []string{
  39                 "",
  40                 "bogustoken",
  41         } {
  42                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  43                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  44                 c.Check(body, check.Equals, "")
  45
  46                 if token != "" {
  47                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  48                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  49                         c.Check(body, check.Equals, "")
  50                 }
  51
  52                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  53                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  54                 c.Check(body, check.Equals, "")
  55         }
  56 }
  57
  58 // TODO: Move most cases to functional tests -- at least use Go's own
  59 // http client instead of forking curl. Just leave enough of an
  60 // integration test to assure that the documented way of invoking curl
  61 // really works against the server.
  62 func (s *IntegrationSuite) Test404(c *check.C) {
  63         for _, uri := range []string{
  64                 // Routing errors (always 404 regardless of what's stored in Keep)
  65                 "/foo",
  66                 "/download",
  67                 "/collections",
  68                 "/collections/",
  69                 // Implicit/generated index is not implemented yet;
  70                 // until then, return 404.
  71                 "/collections/" + arvadostest.FooCollection,
  72                 "/collections/" + arvadostest.FooCollection + "/",
  73                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  74                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  75                 // Non-existent file in collection
  76                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  77                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  78                 // Non-existent collection
  79                 "/collections/" + arvadostest.NonexistentCollection,
  80                 "/collections/" + arvadostest.NonexistentCollection + "/",
  81                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  82                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  83         } {
  84                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  85                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  86                 if len(body) > 0 {
  87                         c.Check(body, check.Equals, "404 page not found\n")
  88                 }
  89         }
  90 }
  91
  92 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  93         if testing.Short() {
  94                 c.Skip("skipping 1GB integration test in short mode")
  95         }
  96         s.test100BlockFile(c, 10000000)
  97 }
  98
  99 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
 100         if testing.Short() {
 101                 // 3 MB
 102                 s.test100BlockFile(c, 30000)
 103         } else {
 104                 // 300 MB
 105                 s.test100BlockFile(c, 3000000)
 106         }
 107 }
 108
 109 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 110         testdata := make([]byte, blocksize)
 111         for i := 0; i < blocksize; i++ {
 112                 testdata[i] = byte(' ')
 113         }
 114         arv, err := arvadosclient.MakeArvadosClient()
 115         c.Assert(err, check.Equals, nil)
 116         arv.ApiToken = arvadostest.ActiveToken
 117         kc, err := keepclient.MakeKeepClient(arv)
 118         c.Assert(err, check.Equals, nil)
 119         loc, _, err := kc.PutB(testdata[:])
 120         c.Assert(err, check.Equals, nil)
 121         mtext := "."
 122         for i := 0; i < 100; i++ {
 123                 mtext = mtext + " " + loc
 124         }
 125         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 126         coll := map[string]interface{}{}
 127         err = arv.Create("collections",
 128                 map[string]interface{}{
 129                         "collection": map[string]interface{}{
 130                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 131                                 "manifest_text": mtext,
 132                         },
 133                 }, &coll)
 134         c.Assert(err, check.Equals, nil)
 135         uuid := coll["uuid"].(string)
 136
 137         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 138         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 139         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 140         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 141         c.Check(size, check.Equals, int64(blocksize)*100)
 142 }
 143
 144 type curlCase struct {
 145         auth    string
 146         host    string
 147         path    string
 148         dataMD5 string
 149 }
 150
 151 func (s *IntegrationSuite) Test200(c *check.C) {
 152         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 153         for _, spec := range []curlCase{
 154                 // My collection
 155                 {
 156                         auth:    arvadostest.ActiveToken,
 157                         host:    arvadostest.FooCollection + "--collections.example.com",
 158                         path:    "/foo",
 159                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 160                 },
 161                 {
 162                         auth:    arvadostest.ActiveToken,
 163                         host:    arvadostest.FooCollection + ".collections.example.com",
 164                         path:    "/foo",
 165                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 166                 },
 167                 {
 168                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
 169                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 170                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 171                 },
 172                 {
 173                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
 174                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 175                 },
 176                 {
 177                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 178                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 179                 },
 180                 {
 181                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 182                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 183                 },
 184                 {
 185                         auth:    "tokensobogus",
 186                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 187                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 188                 },
 189                 {
 190                         auth:    arvadostest.ActiveToken,
 191                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 192                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 193                 },
 194                 {
 195                         auth:    arvadostest.AnonymousToken,
 196                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 197                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 198                 },
 199
 200                 // Anonymously accessible data
 201                 {
 202                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 203                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 204                 },
 205                 {
 206                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 207                         path:    "/Hello%20world.txt",
 208                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 209                 },
 210                 {
 211                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 212                         path:    "/_/Hello%20world.txt",
 213                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 214                 },
 215                 {
 216                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 217                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 218                 },
 219                 {
 220                         auth:    arvadostest.ActiveToken,
 221                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 222                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 223                 },
 224                 {
 225                         auth:    arvadostest.SpectatorToken,
 226                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 227                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 228                 },
 229                 {
 230                         auth:    arvadostest.SpectatorToken,
 231                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 232                         path:    "/Hello%20world.txt",
 233                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 234                 },
 235                 {
 236                         auth:    arvadostest.SpectatorToken,
 237                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 238                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 239                 },
 240         } {
 241                 host := spec.host
 242                 if host == "" {
 243                         host = "collections.example.com"
 244                 }
 245                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 246                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 247                 if strings.HasSuffix(spec.path, ".txt") {
 248                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 249                         // TODO: Check some types that aren't
 250                         // automatically detected by Go's http server
 251                         // by sniffing the content.
 252                 }
 253                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 254         }
 255 }
 256
 257 // Return header block and body.
 258 func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 259         curlArgs := []string{"--silent", "--show-error", "--include"}
 260         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 261         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 262         if token != "" {
 263                 curlArgs = append(curlArgs, "-H", "Authorization: OAuth2 "+token)
 264         }
 265         curlArgs = append(curlArgs, args...)
 266         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 267         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 268         cmd := exec.Command("curl", curlArgs...)
 269         stdout, err := cmd.StdoutPipe()
 270         c.Assert(err, check.IsNil)
 271         cmd.Stderr = os.Stderr
 272         err = cmd.Start()
 273         c.Assert(err, check.IsNil)
 274         buf := make([]byte, 2<<27)
 275         n, err := io.ReadFull(stdout, buf)
 276         // Discard (but measure size of) anything past 128 MiB.
 277         var discarded int64
 278         if err == io.ErrUnexpectedEOF {
 279                 buf = buf[:n]
 280         } else {
 281                 c.Assert(err, check.IsNil)
 282                 discarded, err = io.Copy(ioutil.Discard, stdout)
 283                 c.Assert(err, check.IsNil)
 284         }
 285         err = cmd.Wait()
 286         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 287         // response from the server, even if the response status
 288         // indicates that the request failed. In our test suite, we
 289         // always expect a valid HTTP response, and we parse the
 290         // headers ourselves. If curl exits non-zero, our testing
 291         // environment is broken.
 292         c.Assert(err, check.Equals, nil)
 293         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 294         c.Assert(len(hdrsAndBody), check.Equals, 2)
 295         hdr = hdrsAndBody[0]
 296         bodyPart = hdrsAndBody[1]
 297         bodySize = int64(len(bodyPart)) + discarded
 298         return
 299 }
 300
 301 func (s *IntegrationSuite) TestMetrics(c *check.C) {
 302         origin := "http://" + s.testServer.Addr
 303         req, _ := http.NewRequest("GET", origin+"/notfound", nil)
 304         _, err := http.DefaultClient.Do(req)
 305         c.Assert(err, check.IsNil)
 306         req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
 307         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 308         resp, err := http.DefaultClient.Do(req)
 309         c.Assert(err, check.IsNil)
 310         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 311         for i := 0; i < 2; i++ {
 312                 req, _ = http.NewRequest("GET", origin+"/foo", nil)
 313                 req.Host = arvadostest.FooCollection + ".example.com"
 314                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 315                 resp, err = http.DefaultClient.Do(req)
 316                 c.Assert(err, check.IsNil)
 317                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 318                 buf, _ := ioutil.ReadAll(resp.Body)
 319                 c.Check(buf, check.DeepEquals, []byte("foo"))
 320                 resp.Body.Close()
 321         }
 322
 323         s.testServer.Config.Cache.updateGauges()
 324
 325         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 326         resp, err = http.DefaultClient.Do(req)
 327         c.Assert(err, check.IsNil)
 328         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 329
 330         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 331         req.Header.Set("Authorization", "Bearer badtoken")
 332         resp, err = http.DefaultClient.Do(req)
 333         c.Assert(err, check.IsNil)
 334         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
 335
 336         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 337         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 338         resp, err = http.DefaultClient.Do(req)
 339         c.Assert(err, check.IsNil)
 340         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 341         type summary struct {
 342                 SampleCount string  `json:"sample_count"`
 343                 SampleSum   float64 `json:"sample_sum"`
 344                 Quantile    []struct {
 345                         Quantile float64
 346                         Value    float64
 347                 }
 348         }
 349         type counter struct {
 350                 Value int64
 351         }
 352         type gauge struct {
 353                 Value float64
 354         }
 355         var ents []struct {
 356                 Name   string
 357                 Help   string
 358                 Type   string
 359                 Metric []struct {
 360                         Label []struct {
 361                                 Name  string
 362                                 Value string
 363                         }
 364                         Counter counter
 365                         Gauge   gauge
 366                         Summary summary
 367                 }
 368         }
 369         json.NewDecoder(resp.Body).Decode(&ents)
 370         summaries := map[string]summary{}
 371         gauges := map[string]gauge{}
 372         counters := map[string]counter{}
 373         for _, e := range ents {
 374                 for _, m := range e.Metric {
 375                         labels := map[string]string{}
 376                         for _, lbl := range m.Label {
 377                                 labels[lbl.Name] = lbl.Value
 378                         }
 379                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
 380                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
 381                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 382                 }
 383         }
 384         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
 385         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 386         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 387         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
 388         c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
 389         c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
 390         c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
 391         c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
 392         c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
 393         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
 394         // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
 395         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
 396
 397         // If the Host header indicates a collection, /metrics.json
 398         // refers to a file in the collection -- the metrics handler
 399         // must not intercept that route.
 400         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 401         req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
 402         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 403         resp, err = http.DefaultClient.Do(req)
 404         c.Assert(err, check.IsNil)
 405         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 406 }
 407
 408 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 409         arvadostest.StartAPI()
 410         arvadostest.StartKeep(2, true)
 411
 412         arv, err := arvadosclient.MakeArvadosClient()
 413         c.Assert(err, check.Equals, nil)
 414         arv.ApiToken = arvadostest.ActiveToken
 415         kc, err := keepclient.MakeKeepClient(arv)
 416         c.Assert(err, check.Equals, nil)
 417         kc.PutB([]byte("Hello world\n"))
 418         kc.PutB([]byte("foo"))
 419         kc.PutB([]byte("foobar"))
 420         kc.PutB([]byte("waz"))
 421 }
 422
 423 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 424         arvadostest.StopKeep(2)
 425         arvadostest.StopAPI()
 426 }
 427
 428 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 429         arvadostest.ResetEnv()
 430         ldr := config.NewLoader(nil, nil)
 431         arvCfg, err := ldr.LoadDefaults()
 432         cfg := DefaultConfig(arvCfg)
 433         c.Assert(err, check.IsNil)
 434         cfg.Client = arvados.Client{
 435                 APIHost:  testAPIHost,
 436                 Insecure: true,
 437         }
 438         listen := "127.0.0.1:0"
 439         cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
 440         cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
 441         cfg.cluster.ManagementToken = arvadostest.ManagementToken
 442         s.testServer = &server{Config: cfg}
 443         err = s.testServer.Start()
 444         c.Assert(err, check.Equals, nil)
 445 }
 446
 447 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 448         var err error
 449         if s.testServer != nil {
 450                 err = s.testServer.Close()
 451         }
 452         c.Check(err, check.Equals, nil)
 453 }
 454
 455 // Gocheck boilerplate
 456 func Test(t *testing.T) {
 457         check.TestingT(t)
 458 }