services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "encoding/json"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "net"
  16         "net/http"
  17         "net/http/httptest"
  18         "os"
  19         "os/exec"
  20         "regexp"
  21         "strings"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/config"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/httpserver"
  31         "git.arvados.org/arvados.git/sdk/go/keepclient"
  32         "github.com/prometheus/client_golang/prometheus"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  37
  38 var _ = check.Suite(&IntegrationSuite{})
  39
  40 // IntegrationSuite tests need an API server and a keep-web server
  41 type IntegrationSuite struct {
  42         testServer *httptest.Server
  43         handler    *handler
  44 }
  45
  46 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  47         for _, token := range []string{
  48                 "",
  49                 "bogustoken",
  50         } {
  51                 hdr, body, _ := s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/c="+arvadostest.FooCollection+"/foo")
  52                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
  53                 c.Check(strings.TrimSpace(body), check.Equals, unauthorizedMessage)
  54
  55                 if token != "" {
  56                         hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  57                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  58                         c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
  59                 }
  60
  61                 hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/bad-route")
  62                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  63                 c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
  64         }
  65 }
  66
  67 // TODO: Move most cases to functional tests -- at least use Go's own
  68 // http client instead of forking curl. Just leave enough of an
  69 // integration test to assure that the documented way of invoking curl
  70 // really works against the server.
  71 func (s *IntegrationSuite) Test404(c *check.C) {
  72         for _, uri := range []string{
  73                 // Routing errors (always 404 regardless of what's stored in Keep)
  74                 "/foo",
  75                 "/download",
  76                 "/collections",
  77                 "/collections/",
  78                 // Non-existent file/directory
  79                 "/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  80                 "/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy/",
  81                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  82                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy/",
  83                 // Non-existent collection
  84                 "/c=" + arvadostest.NonexistentCollection,
  85                 "/c=" + arvadostest.NonexistentCollection + "/",
  86                 "/c=" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  87                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  88         } {
  89                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, uri)
  90                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  91                 if len(body) > 0 {
  92                         c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
  93                 }
  94         }
  95 }
  96
  97 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  98         if testing.Short() {
  99                 c.Skip("skipping 1GB integration test in short mode")
 100         }
 101         s.test100BlockFile(c, 10000000)
 102 }
 103
 104 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
 105         if testing.Short() {
 106                 // 3 MB
 107                 s.test100BlockFile(c, 30000)
 108         } else {
 109                 // 300 MB
 110                 s.test100BlockFile(c, 3000000)
 111         }
 112 }
 113
 114 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 115         testdata := make([]byte, blocksize)
 116         for i := 0; i < blocksize; i++ {
 117                 testdata[i] = byte(' ')
 118         }
 119         arv, err := arvadosclient.MakeArvadosClient()
 120         c.Assert(err, check.Equals, nil)
 121         arv.ApiToken = arvadostest.ActiveToken
 122         kc, err := keepclient.MakeKeepClient(arv)
 123         c.Assert(err, check.Equals, nil)
 124         loc, _, err := kc.PutB(testdata[:])
 125         c.Assert(err, check.Equals, nil)
 126         mtext := "."
 127         for i := 0; i < 100; i++ {
 128                 mtext = mtext + " " + loc
 129         }
 130         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 131         coll := map[string]interface{}{}
 132         err = arv.Create("collections",
 133                 map[string]interface{}{
 134                         "collection": map[string]interface{}{
 135                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 136                                 "manifest_text": mtext,
 137                         },
 138                 }, &coll)
 139         c.Assert(err, check.Equals, nil)
 140         uuid := coll["uuid"].(string)
 141
 142         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 143         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 144         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 145         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 146         c.Check(size, check.Equals, int64(blocksize)*100)
 147 }
 148
 149 type curlCase struct {
 150         auth    string
 151         host    string
 152         path    string
 153         dataMD5 string
 154 }
 155
 156 func (s *IntegrationSuite) Test200(c *check.C) {
 157         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 158         for _, spec := range []curlCase{
 159                 // My collection
 160                 {
 161                         auth:    arvadostest.ActiveToken,
 162                         host:    arvadostest.FooCollection + "--collections.example.com",
 163                         path:    "/foo",
 164                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 165                 },
 166                 {
 167                         auth:    arvadostest.ActiveToken,
 168                         host:    arvadostest.FooCollection + ".collections.example.com",
 169                         path:    "/foo",
 170                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 171                 },
 172                 {
 173                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
 174                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 175                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 176                 },
 177                 {
 178                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
 179                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 180                 },
 181                 {
 182                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 183                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 184                 },
 185                 {
 186                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 187                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 188                 },
 189                 {
 190                         auth:    "tokensobogus",
 191                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 192                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 193                 },
 194                 {
 195                         auth:    arvadostest.ActiveToken,
 196                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 197                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 198                 },
 199                 {
 200                         auth:    arvadostest.AnonymousToken,
 201                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 202                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 203                 },
 204
 205                 // Anonymously accessible data
 206                 {
 207                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 208                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 209                 },
 210                 {
 211                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 212                         path:    "/Hello%20world.txt",
 213                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 214                 },
 215                 {
 216                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 217                         path:    "/_/Hello%20world.txt",
 218                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 219                 },
 220                 {
 221                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 222                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 223                 },
 224                 {
 225                         auth:    arvadostest.ActiveToken,
 226                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 227                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 228                 },
 229                 {
 230                         auth:    arvadostest.SpectatorToken,
 231                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 232                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 233                 },
 234                 {
 235                         auth:    arvadostest.SpectatorToken,
 236                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 237                         path:    "/Hello%20world.txt",
 238                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 239                 },
 240                 {
 241                         auth:    arvadostest.SpectatorToken,
 242                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 243                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 244                 },
 245         } {
 246                 host := spec.host
 247                 if host == "" {
 248                         host = "collections.example.com"
 249                 }
 250                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 251                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 252                 if strings.HasSuffix(spec.path, ".txt") {
 253                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 254                         // TODO: Check some types that aren't
 255                         // automatically detected by Go's http server
 256                         // by sniffing the content.
 257                 }
 258                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 259         }
 260 }
 261
 262 // Return header block and body.
 263 func (s *IntegrationSuite) runCurl(c *check.C, auth, hostport, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 264         curlArgs := []string{"--silent", "--show-error", "--include"}
 265         testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
 266         host, port, _ := net.SplitHostPort(hostport)
 267         if port == "" {
 268                 port = "80"
 269         }
 270         curlArgs = append(curlArgs, "--connect-to", host+":"+port+":"+testHost+":"+testPort)
 271         if strings.Contains(auth, " ") {
 272                 // caller supplied entire Authorization header value
 273                 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
 274         } else if auth != "" {
 275                 // caller supplied Arvados token
 276                 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 277         }
 278         curlArgs = append(curlArgs, args...)
 279         curlArgs = append(curlArgs, "http://"+hostport+uri)
 280         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 281         cmd := exec.Command("curl", curlArgs...)
 282         stdout, err := cmd.StdoutPipe()
 283         c.Assert(err, check.IsNil)
 284         cmd.Stderr = os.Stderr
 285         err = cmd.Start()
 286         c.Assert(err, check.IsNil)
 287         buf := make([]byte, 2<<27)
 288         n, err := io.ReadFull(stdout, buf)
 289         // Discard (but measure size of) anything past 128 MiB.
 290         var discarded int64
 291         if err == io.ErrUnexpectedEOF {
 292                 buf = buf[:n]
 293         } else {
 294                 c.Assert(err, check.IsNil)
 295                 discarded, err = io.Copy(ioutil.Discard, stdout)
 296                 c.Assert(err, check.IsNil)
 297         }
 298         err = cmd.Wait()
 299         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 300         // response from the server, even if the response status
 301         // indicates that the request failed. In our test suite, we
 302         // always expect a valid HTTP response, and we parse the
 303         // headers ourselves. If curl exits non-zero, our testing
 304         // environment is broken.
 305         c.Assert(err, check.Equals, nil)
 306         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 307         c.Assert(len(hdrsAndBody), check.Equals, 2)
 308         hdr = hdrsAndBody[0]
 309         bodyPart = hdrsAndBody[1]
 310         bodySize = int64(len(bodyPart)) + discarded
 311         return
 312 }
 313
 314 // Run a full-featured server, including the metrics/health routes
 315 // that are added by service.Command.
 316 func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
 317         logbuf = &bytes.Buffer{}
 318         cluster = *s.handler.Cluster
 319         cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
 320         cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
 321
 322         var configjson bytes.Buffer
 323         json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
 324         go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
 325         for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
 326                 if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
 327                         srvaddr = "http://" + m[1]
 328                         break
 329                 }
 330         }
 331         if srvaddr == "" {
 332                 c.Fatal("timed out")
 333         }
 334         return
 335 }
 336
 337 // Ensure uploads can take longer than API.RequestTimeout.
 338 //
 339 // Currently, this works only by accident: service.Command cancels the
 340 // request context as usual (there is no exemption), but
 341 // webdav.Handler doesn't notice if the request context is cancelled
 342 // while waiting to send or receive file data.
 343 func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
 344         s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
 345         _, srvaddr, _ := s.runServer(c)
 346
 347         var coll arvados.Collection
 348         arv, err := arvadosclient.MakeArvadosClient()
 349         c.Assert(err, check.IsNil)
 350         arv.ApiToken = arvadostest.ActiveTokenV2
 351         err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
 352         c.Assert(err, check.IsNil)
 353
 354         pr, pw := io.Pipe()
 355         go func() {
 356                 time.Sleep(time.Second)
 357                 pw.Write(make([]byte, 10000000))
 358                 pw.Close()
 359         }()
 360         req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
 361         req.Host = coll.UUID + ".example"
 362         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 363         resp, err := http.DefaultClient.Do(req)
 364         c.Assert(err, check.IsNil)
 365         c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
 366
 367         req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
 368         req.Host = coll.UUID + ".example"
 369         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 370         resp, err = http.DefaultClient.Do(req)
 371         c.Assert(err, check.IsNil)
 372         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 373         time.Sleep(time.Second)
 374         body, err := ioutil.ReadAll(resp.Body)
 375         c.Check(err, check.IsNil)
 376         c.Check(len(body), check.Equals, 10000000)
 377 }
 378
 379 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 380         cluster, srvaddr, _ := s.runServer(c)
 381         req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
 382         req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
 383         resp, err := http.DefaultClient.Do(req)
 384         c.Assert(err, check.IsNil)
 385         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 386         body, _ := ioutil.ReadAll(resp.Body)
 387         c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
 388 }
 389
 390 func (s *IntegrationSuite) TestMetrics(c *check.C) {
 391         cluster, srvaddr, _ := s.runServer(c)
 392
 393         req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
 394         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 395         _, err := http.DefaultClient.Do(req)
 396         c.Assert(err, check.IsNil)
 397         req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
 398         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 399         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 400         resp, err := http.DefaultClient.Do(req)
 401         c.Assert(err, check.IsNil)
 402         c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
 403         for i := 0; i < 2; i++ {
 404                 req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
 405                 req.Host = arvadostest.FooCollection + ".example.com"
 406                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 407                 resp, err = http.DefaultClient.Do(req)
 408                 c.Assert(err, check.IsNil)
 409                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 410                 buf, _ := ioutil.ReadAll(resp.Body)
 411                 c.Check(buf, check.DeepEquals, []byte("foo"))
 412                 resp.Body.Close()
 413         }
 414
 415         var coll arvados.Collection
 416         arv, err := arvadosclient.MakeArvadosClient()
 417         c.Assert(err, check.IsNil)
 418         arv.ApiToken = arvadostest.ActiveTokenV2
 419         err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
 420         c.Assert(err, check.IsNil)
 421         defer arv.Delete("collections", coll.UUID, nil, nil)
 422         for i := 0; i < 2; i++ {
 423                 size := 1 << (i * 12)
 424                 req, _ = http.NewRequest("PUT", srvaddr+"/zero-"+fmt.Sprintf("%d", size), bytes.NewReader(make([]byte, size)))
 425                 req.Host = coll.UUID + ".example.com"
 426                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 427                 resp, err = http.DefaultClient.Do(req)
 428                 c.Assert(err, check.IsNil)
 429                 c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
 430                 resp.Body.Close()
 431         }
 432
 433         time.Sleep(metricsUpdateInterval * 2)
 434
 435         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 436         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 437         resp, err = http.DefaultClient.Do(req)
 438         c.Assert(err, check.IsNil)
 439         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 440
 441         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 442         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 443         req.Header.Set("Authorization", "Bearer badtoken")
 444         resp, err = http.DefaultClient.Do(req)
 445         c.Assert(err, check.IsNil)
 446         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
 447
 448         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 449         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 450         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 451         resp, err = http.DefaultClient.Do(req)
 452         c.Assert(err, check.IsNil)
 453         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 454         type summary struct {
 455                 SampleCount string
 456                 SampleSum   float64
 457         }
 458         type counter struct {
 459                 Value int64
 460         }
 461         type gauge struct {
 462                 Value float64
 463         }
 464         var ents []struct {
 465                 Name   string
 466                 Help   string
 467                 Type   string
 468                 Metric []struct {
 469                         Label []struct {
 470                                 Name  string
 471                                 Value string
 472                         }
 473                         Counter counter
 474                         Gauge   gauge
 475                         Summary summary
 476                 }
 477         }
 478         json.NewDecoder(resp.Body).Decode(&ents)
 479         summaries := map[string]summary{}
 480         gauges := map[string]gauge{}
 481         counters := map[string]counter{}
 482         for _, e := range ents {
 483                 for _, m := range e.Metric {
 484                         labels := map[string]string{}
 485                         for _, lbl := range m.Label {
 486                                 labels[lbl.Name] = lbl.Value
 487                         }
 488                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
 489                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
 490                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 491                 }
 492         }
 493         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
 494         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 495         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 496         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
 497         c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(1208))
 498
 499         // If the Host header indicates a collection, /metrics.json
 500         // refers to a file in the collection -- the metrics handler
 501         // must not intercept that route. Ditto health check paths.
 502         for _, path := range []string{"/metrics.json", "/_health/ping"} {
 503                 c.Logf("path: %q", path)
 504                 req, _ = http.NewRequest("GET", srvaddr+path, nil)
 505                 req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
 506                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 507                 resp, err = http.DefaultClient.Do(req)
 508                 c.Assert(err, check.IsNil)
 509                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 510         }
 511
 512         req, _ = http.NewRequest("GET", srvaddr+"/metrics", nil)
 513         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 514         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 515         resp, err = http.DefaultClient.Do(req)
 516         c.Assert(err, check.IsNil)
 517         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 518         allmetrics, err := ioutil.ReadAll(resp.Body)
 519         c.Check(err, check.IsNil)
 520
 521         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_apparent_backend_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
 522         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
 523         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_speed_bucket{size_range="0",le="\+Inf"} 2\n.*`)
 524         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_sync_delay_seconds_bucket{size_range="0",le="10"} 2\n.*`)
 525
 526         c.Logf("%s", allmetrics)
 527 }
 528
 529 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 530         arvadostest.ResetDB(c)
 531         arvadostest.StartKeep(2, true)
 532
 533         arv, err := arvadosclient.MakeArvadosClient()
 534         c.Assert(err, check.Equals, nil)
 535         arv.ApiToken = arvadostest.ActiveToken
 536         kc, err := keepclient.MakeKeepClient(arv)
 537         c.Assert(err, check.Equals, nil)
 538         kc.PutB([]byte("Hello world\n"))
 539         kc.PutB([]byte("foo"))
 540         kc.PutB([]byte("foobar"))
 541         kc.PutB([]byte("waz"))
 542 }
 543
 544 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 545         arvadostest.StopKeep(2)
 546 }
 547
 548 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 549         arvadostest.ResetEnv()
 550         logger := ctxlog.TestLogger(c)
 551         ldr := config.NewLoader(&bytes.Buffer{}, logger)
 552         cfg, err := ldr.Load()
 553         c.Assert(err, check.IsNil)
 554         cluster, err := cfg.GetCluster("")
 555         c.Assert(err, check.IsNil)
 556
 557         ctx := ctxlog.Context(context.Background(), logger)
 558
 559         s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, prometheus.NewRegistry()).(*handler)
 560         s.testServer = httptest.NewUnstartedServer(
 561                 httpserver.AddRequestIDs(
 562                         httpserver.LogRequests(
 563                                 s.handler)))
 564         s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
 565         s.testServer.Start()
 566
 567         cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 568         cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 569 }
 570
 571 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 572         if s.testServer != nil {
 573                 s.testServer.Close()
 574         }
 575 }
 576
 577 // Gocheck boilerplate
 578 func Test(t *testing.T) {
 579         check.TestingT(t)
 580 }