services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package main
   6
   7 import (
   8         "bytes"
   9         "crypto/md5"
  10         "encoding/json"
  11         "fmt"
  12         "io"
  13         "io/ioutil"
  14         "net"
  15         "net/http"
  16         "os"
  17         "os/exec"
  18         "strings"
  19         "testing"
  20
  21         "git.arvados.org/arvados.git/lib/config"
  22         "git.arvados.org/arvados.git/sdk/go/arvados"
  23         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  24         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  25         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  26         "git.arvados.org/arvados.git/sdk/go/keepclient"
  27         check "gopkg.in/check.v1"
  28 )
  29
  30 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  31
  32 var _ = check.Suite(&IntegrationSuite{})
  33
  34 // IntegrationSuite tests need an API server and a keep-web server
  35 type IntegrationSuite struct {
  36         testServer *server
  37 }
  38
  39 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  40         for _, token := range []string{
  41                 "",
  42                 "bogustoken",
  43         } {
  44                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  45                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  46                 c.Check(body, check.Equals, notFoundMessage+"\n")
  47
  48                 if token != "" {
  49                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  50                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  51                         c.Check(body, check.Equals, notFoundMessage+"\n")
  52                 }
  53
  54                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  55                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  56                 c.Check(body, check.Equals, notFoundMessage+"\n")
  57         }
  58 }
  59
  60 // TODO: Move most cases to functional tests -- at least use Go's own
  61 // http client instead of forking curl. Just leave enough of an
  62 // integration test to assure that the documented way of invoking curl
  63 // really works against the server.
  64 func (s *IntegrationSuite) Test404(c *check.C) {
  65         for _, uri := range []string{
  66                 // Routing errors (always 404 regardless of what's stored in Keep)
  67                 "/foo",
  68                 "/download",
  69                 "/collections",
  70                 "/collections/",
  71                 // Implicit/generated index is not implemented yet;
  72                 // until then, return 404.
  73                 "/collections/" + arvadostest.FooCollection,
  74                 "/collections/" + arvadostest.FooCollection + "/",
  75                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  76                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  77                 // Non-existent file in collection
  78                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  79                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  80                 // Non-existent collection
  81                 "/collections/" + arvadostest.NonexistentCollection,
  82                 "/collections/" + arvadostest.NonexistentCollection + "/",
  83                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  84                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  85         } {
  86                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  87                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  88                 if len(body) > 0 {
  89                         c.Check(body, check.Equals, notFoundMessage+"\n")
  90                 }
  91         }
  92 }
  93
  94 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  95         if testing.Short() {
  96                 c.Skip("skipping 1GB integration test in short mode")
  97         }
  98         s.test100BlockFile(c, 10000000)
  99 }
 100
 101 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
 102         if testing.Short() {
 103                 // 3 MB
 104                 s.test100BlockFile(c, 30000)
 105         } else {
 106                 // 300 MB
 107                 s.test100BlockFile(c, 3000000)
 108         }
 109 }
 110
 111 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 112         testdata := make([]byte, blocksize)
 113         for i := 0; i < blocksize; i++ {
 114                 testdata[i] = byte(' ')
 115         }
 116         arv, err := arvadosclient.MakeArvadosClient()
 117         c.Assert(err, check.Equals, nil)
 118         arv.ApiToken = arvadostest.ActiveToken
 119         kc, err := keepclient.MakeKeepClient(arv)
 120         c.Assert(err, check.Equals, nil)
 121         loc, _, err := kc.PutB(testdata[:])
 122         c.Assert(err, check.Equals, nil)
 123         mtext := "."
 124         for i := 0; i < 100; i++ {
 125                 mtext = mtext + " " + loc
 126         }
 127         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 128         coll := map[string]interface{}{}
 129         err = arv.Create("collections",
 130                 map[string]interface{}{
 131                         "collection": map[string]interface{}{
 132                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 133                                 "manifest_text": mtext,
 134                         },
 135                 }, &coll)
 136         c.Assert(err, check.Equals, nil)
 137         uuid := coll["uuid"].(string)
 138
 139         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 140         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 141         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 142         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 143         c.Check(size, check.Equals, int64(blocksize)*100)
 144 }
 145
 146 type curlCase struct {
 147         auth    string
 148         host    string
 149         path    string
 150         dataMD5 string
 151 }
 152
 153 func (s *IntegrationSuite) Test200(c *check.C) {
 154         s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 155         for _, spec := range []curlCase{
 156                 // My collection
 157                 {
 158                         auth:    arvadostest.ActiveToken,
 159                         host:    arvadostest.FooCollection + "--collections.example.com",
 160                         path:    "/foo",
 161                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 162                 },
 163                 {
 164                         auth:    arvadostest.ActiveToken,
 165                         host:    arvadostest.FooCollection + ".collections.example.com",
 166                         path:    "/foo",
 167                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 168                 },
 169                 {
 170                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
 171                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 172                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 173                 },
 174                 {
 175                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
 176                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 177                 },
 178                 {
 179                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 180                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 181                 },
 182                 {
 183                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 184                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 185                 },
 186                 {
 187                         auth:    "tokensobogus",
 188                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 189                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 190                 },
 191                 {
 192                         auth:    arvadostest.ActiveToken,
 193                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 194                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 195                 },
 196                 {
 197                         auth:    arvadostest.AnonymousToken,
 198                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 199                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 200                 },
 201
 202                 // Anonymously accessible data
 203                 {
 204                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 205                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 206                 },
 207                 {
 208                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 209                         path:    "/Hello%20world.txt",
 210                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 211                 },
 212                 {
 213                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 214                         path:    "/_/Hello%20world.txt",
 215                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 216                 },
 217                 {
 218                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 219                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 220                 },
 221                 {
 222                         auth:    arvadostest.ActiveToken,
 223                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 224                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 225                 },
 226                 {
 227                         auth:    arvadostest.SpectatorToken,
 228                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 229                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 230                 },
 231                 {
 232                         auth:    arvadostest.SpectatorToken,
 233                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 234                         path:    "/Hello%20world.txt",
 235                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 236                 },
 237                 {
 238                         auth:    arvadostest.SpectatorToken,
 239                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 240                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 241                 },
 242         } {
 243                 host := spec.host
 244                 if host == "" {
 245                         host = "collections.example.com"
 246                 }
 247                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 248                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 249                 if strings.HasSuffix(spec.path, ".txt") {
 250                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 251                         // TODO: Check some types that aren't
 252                         // automatically detected by Go's http server
 253                         // by sniffing the content.
 254                 }
 255                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 256         }
 257 }
 258
 259 // Return header block and body.
 260 func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 261         curlArgs := []string{"--silent", "--show-error", "--include"}
 262         testHost, testPort, _ := net.SplitHostPort(s.testServer.Addr)
 263         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 264         if strings.Contains(auth, " ") {
 265                 // caller supplied entire Authorization header value
 266                 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
 267         } else if auth != "" {
 268                 // caller supplied Arvados token
 269                 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 270         }
 271         curlArgs = append(curlArgs, args...)
 272         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 273         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 274         cmd := exec.Command("curl", curlArgs...)
 275         stdout, err := cmd.StdoutPipe()
 276         c.Assert(err, check.IsNil)
 277         cmd.Stderr = os.Stderr
 278         err = cmd.Start()
 279         c.Assert(err, check.IsNil)
 280         buf := make([]byte, 2<<27)
 281         n, err := io.ReadFull(stdout, buf)
 282         // Discard (but measure size of) anything past 128 MiB.
 283         var discarded int64
 284         if err == io.ErrUnexpectedEOF {
 285                 buf = buf[:n]
 286         } else {
 287                 c.Assert(err, check.IsNil)
 288                 discarded, err = io.Copy(ioutil.Discard, stdout)
 289                 c.Assert(err, check.IsNil)
 290         }
 291         err = cmd.Wait()
 292         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 293         // response from the server, even if the response status
 294         // indicates that the request failed. In our test suite, we
 295         // always expect a valid HTTP response, and we parse the
 296         // headers ourselves. If curl exits non-zero, our testing
 297         // environment is broken.
 298         c.Assert(err, check.Equals, nil)
 299         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 300         c.Assert(len(hdrsAndBody), check.Equals, 2)
 301         hdr = hdrsAndBody[0]
 302         bodyPart = hdrsAndBody[1]
 303         bodySize = int64(len(bodyPart)) + discarded
 304         return
 305 }
 306
 307 func (s *IntegrationSuite) TestMetrics(c *check.C) {
 308         s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr
 309         origin := "http://" + s.testServer.Addr
 310         req, _ := http.NewRequest("GET", origin+"/notfound", nil)
 311         _, err := http.DefaultClient.Do(req)
 312         c.Assert(err, check.IsNil)
 313         req, _ = http.NewRequest("GET", origin+"/by_id/", nil)
 314         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 315         resp, err := http.DefaultClient.Do(req)
 316         c.Assert(err, check.IsNil)
 317         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 318         for i := 0; i < 2; i++ {
 319                 req, _ = http.NewRequest("GET", origin+"/foo", nil)
 320                 req.Host = arvadostest.FooCollection + ".example.com"
 321                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 322                 resp, err = http.DefaultClient.Do(req)
 323                 c.Assert(err, check.IsNil)
 324                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 325                 buf, _ := ioutil.ReadAll(resp.Body)
 326                 c.Check(buf, check.DeepEquals, []byte("foo"))
 327                 resp.Body.Close()
 328         }
 329
 330         s.testServer.Config.Cache.updateGauges()
 331
 332         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 333         resp, err = http.DefaultClient.Do(req)
 334         c.Assert(err, check.IsNil)
 335         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 336
 337         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 338         req.Header.Set("Authorization", "Bearer badtoken")
 339         resp, err = http.DefaultClient.Do(req)
 340         c.Assert(err, check.IsNil)
 341         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
 342
 343         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 344         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 345         resp, err = http.DefaultClient.Do(req)
 346         c.Assert(err, check.IsNil)
 347         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 348         type summary struct {
 349                 SampleCount string
 350                 SampleSum   float64
 351         }
 352         type counter struct {
 353                 Value int64
 354         }
 355         type gauge struct {
 356                 Value float64
 357         }
 358         var ents []struct {
 359                 Name   string
 360                 Help   string
 361                 Type   string
 362                 Metric []struct {
 363                         Label []struct {
 364                                 Name  string
 365                                 Value string
 366                         }
 367                         Counter counter
 368                         Gauge   gauge
 369                         Summary summary
 370                 }
 371         }
 372         json.NewDecoder(resp.Body).Decode(&ents)
 373         summaries := map[string]summary{}
 374         gauges := map[string]gauge{}
 375         counters := map[string]counter{}
 376         for _, e := range ents {
 377                 for _, m := range e.Metric {
 378                         labels := map[string]string{}
 379                         for _, lbl := range m.Label {
 380                                 labels[lbl.Name] = lbl.Value
 381                         }
 382                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
 383                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
 384                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 385                 }
 386         }
 387         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
 388         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 389         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 390         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
 391         c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
 392         c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1))
 393         c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
 394         c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
 395         c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1))
 396         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
 397         // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
 398         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51))
 399
 400         // If the Host header indicates a collection, /metrics.json
 401         // refers to a file in the collection -- the metrics handler
 402         // must not intercept that route.
 403         req, _ = http.NewRequest("GET", origin+"/metrics.json", nil)
 404         req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
 405         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 406         resp, err = http.DefaultClient.Do(req)
 407         c.Assert(err, check.IsNil)
 408         c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 409 }
 410
 411 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 412         arvadostest.StartAPI()
 413         arvadostest.StartKeep(2, true)
 414
 415         arv, err := arvadosclient.MakeArvadosClient()
 416         c.Assert(err, check.Equals, nil)
 417         arv.ApiToken = arvadostest.ActiveToken
 418         kc, err := keepclient.MakeKeepClient(arv)
 419         c.Assert(err, check.Equals, nil)
 420         kc.PutB([]byte("Hello world\n"))
 421         kc.PutB([]byte("foo"))
 422         kc.PutB([]byte("foobar"))
 423         kc.PutB([]byte("waz"))
 424 }
 425
 426 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 427         arvadostest.StopKeep(2)
 428         arvadostest.StopAPI()
 429 }
 430
 431 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 432         arvadostest.ResetEnv()
 433         ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c))
 434         ldr.Path = "-"
 435         arvCfg, err := ldr.Load()
 436         c.Check(err, check.IsNil)
 437         cfg := newConfig(arvCfg)
 438         c.Assert(err, check.IsNil)
 439         cfg.Client = arvados.Client{
 440                 APIHost:  testAPIHost,
 441                 Insecure: true,
 442         }
 443         listen := "127.0.0.1:0"
 444         cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
 445         cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{}
 446         cfg.cluster.ManagementToken = arvadostest.ManagementToken
 447         cfg.cluster.SystemRootToken = arvadostest.SystemRootToken
 448         cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 449         s.testServer = &server{Config: cfg}
 450         err = s.testServer.Start(ctxlog.TestLogger(c))
 451         c.Assert(err, check.Equals, nil)
 452 }
 453
 454 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 455         var err error
 456         if s.testServer != nil {
 457                 err = s.testServer.Close()
 458         }
 459         c.Check(err, check.Equals, nil)
 460 }
 461
 462 // Gocheck boilerplate
 463 func Test(t *testing.T) {
 464         check.TestingT(t)
 465 }