services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "encoding/json"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "net"
  16         "net/http"
  17         "net/http/httptest"
  18         "os"
  19         "os/exec"
  20         "regexp"
  21         "strings"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/config"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/httpserver"
  31         "git.arvados.org/arvados.git/sdk/go/keepclient"
  32         "github.com/prometheus/client_golang/prometheus"
  33         check "gopkg.in/check.v1"
  34 )
  35
  36 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  37
  38 var _ = check.Suite(&IntegrationSuite{})
  39
  40 // IntegrationSuite tests need an API server and a keep-web server
  41 type IntegrationSuite struct {
  42         testServer *httptest.Server
  43         handler    *handler
  44         ctx        context.Context
  45 }
  46
  47 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  48         for _, token := range []string{
  49                 "",
  50                 "bogustoken",
  51         } {
  52                 hdr, body, _ := s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/c="+arvadostest.FooCollection+"/foo")
  53                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
  54                 c.Check(strings.TrimSpace(body), check.Equals, unauthorizedMessage)
  55
  56                 if token != "" {
  57                         hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  58                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  59                         c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
  60                 }
  61
  62                 hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/bad-route")
  63                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  64                 c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
  65         }
  66 }
  67
  68 // TODO: Move most cases to functional tests -- at least use Go's own
  69 // http client instead of forking curl. Just leave enough of an
  70 // integration test to assure that the documented way of invoking curl
  71 // really works against the server.
  72 func (s *IntegrationSuite) Test404(c *check.C) {
  73         for _, uri := range []string{
  74                 // Routing errors (always 404 regardless of what's stored in Keep)
  75                 "/foo",
  76                 "/download",
  77                 "/collections",
  78                 "/collections/",
  79                 // Non-existent file/directory
  80                 "/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  81                 "/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy/",
  82                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  83                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy/",
  84                 // Non-existent collection
  85                 "/c=" + arvadostest.NonexistentCollection,
  86                 "/c=" + arvadostest.NonexistentCollection + "/",
  87                 "/c=" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  88                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  89         } {
  90                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, uri)
  91                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  92                 if len(body) > 0 {
  93                         c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
  94                 }
  95         }
  96 }
  97
  98 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
  99         if testing.Short() {
 100                 c.Skip("skipping 1GB integration test in short mode")
 101         }
 102         s.test100BlockFile(c, 10000000)
 103 }
 104
 105 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
 106         if testing.Short() {
 107                 // 3 MB
 108                 s.test100BlockFile(c, 30000)
 109         } else {
 110                 // 300 MB
 111                 s.test100BlockFile(c, 3000000)
 112         }
 113 }
 114
 115 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 116         testdata := make([]byte, blocksize)
 117         for i := 0; i < blocksize; i++ {
 118                 testdata[i] = byte(' ')
 119         }
 120         arv, err := arvadosclient.MakeArvadosClient()
 121         c.Assert(err, check.Equals, nil)
 122         arv.ApiToken = arvadostest.ActiveToken
 123         kc, err := keepclient.MakeKeepClient(arv)
 124         c.Assert(err, check.Equals, nil)
 125         loc, _, err := kc.PutB(testdata[:])
 126         c.Assert(err, check.Equals, nil)
 127         mtext := "."
 128         for i := 0; i < 100; i++ {
 129                 mtext = mtext + " " + loc
 130         }
 131         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 132         coll := map[string]interface{}{}
 133         err = arv.Create("collections",
 134                 map[string]interface{}{
 135                         "collection": map[string]interface{}{
 136                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 137                                 "manifest_text": mtext,
 138                         },
 139                 }, &coll)
 140         c.Assert(err, check.Equals, nil)
 141         uuid := coll["uuid"].(string)
 142
 143         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 144         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 145         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 146         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 147         c.Check(size, check.Equals, int64(blocksize)*100)
 148 }
 149
 150 type curlCase struct {
 151         auth    string
 152         host    string
 153         path    string
 154         dataMD5 string
 155 }
 156
 157 func (s *IntegrationSuite) Test200(c *check.C) {
 158         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 159         for _, spec := range []curlCase{
 160                 // My collection
 161                 {
 162                         auth:    arvadostest.ActiveToken,
 163                         host:    arvadostest.FooCollection + "--collections.example.com",
 164                         path:    "/foo",
 165                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 166                 },
 167                 {
 168                         auth:    arvadostest.ActiveToken,
 169                         host:    arvadostest.FooCollection + ".collections.example.com",
 170                         path:    "/foo",
 171                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 172                 },
 173                 {
 174                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
 175                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 176                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 177                 },
 178                 {
 179                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
 180                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 181                 },
 182                 {
 183                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 184                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 185                 },
 186                 {
 187                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 188                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 189                 },
 190                 {
 191                         auth:    "tokensobogus",
 192                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 193                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 194                 },
 195                 {
 196                         auth:    arvadostest.ActiveToken,
 197                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 198                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 199                 },
 200                 {
 201                         auth:    arvadostest.AnonymousToken,
 202                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 203                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 204                 },
 205
 206                 // Anonymously accessible data
 207                 {
 208                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 209                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 210                 },
 211                 {
 212                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 213                         path:    "/Hello%20world.txt",
 214                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 215                 },
 216                 {
 217                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 218                         path:    "/_/Hello%20world.txt",
 219                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 220                 },
 221                 {
 222                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 223                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 224                 },
 225                 {
 226                         auth:    arvadostest.ActiveToken,
 227                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 228                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 229                 },
 230                 {
 231                         auth:    arvadostest.SpectatorToken,
 232                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 233                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 234                 },
 235                 {
 236                         auth:    arvadostest.SpectatorToken,
 237                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 238                         path:    "/Hello%20world.txt",
 239                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 240                 },
 241                 {
 242                         auth:    arvadostest.SpectatorToken,
 243                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 244                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 245                 },
 246         } {
 247                 host := spec.host
 248                 if host == "" {
 249                         host = "collections.example.com"
 250                 }
 251                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 252                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 253                 if strings.HasSuffix(spec.path, ".txt") {
 254                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 255                         // TODO: Check some types that aren't
 256                         // automatically detected by Go's http server
 257                         // by sniffing the content.
 258                 }
 259                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 260         }
 261 }
 262
 263 // Return header block and body.
 264 func (s *IntegrationSuite) runCurl(c *check.C, auth, hostport, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 265         curlArgs := []string{"--silent", "--show-error", "--include"}
 266         testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
 267         host, port, _ := net.SplitHostPort(hostport)
 268         if port == "" {
 269                 port = "80"
 270         }
 271         curlArgs = append(curlArgs, "--connect-to", host+":"+port+":"+testHost+":"+testPort)
 272         if strings.Contains(auth, " ") {
 273                 // caller supplied entire Authorization header value
 274                 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
 275         } else if auth != "" {
 276                 // caller supplied Arvados token
 277                 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 278         }
 279         curlArgs = append(curlArgs, args...)
 280         curlArgs = append(curlArgs, "http://"+hostport+uri)
 281         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 282         cmd := exec.Command("curl", curlArgs...)
 283         stdout, err := cmd.StdoutPipe()
 284         c.Assert(err, check.IsNil)
 285         cmd.Stderr = os.Stderr
 286         err = cmd.Start()
 287         c.Assert(err, check.IsNil)
 288         buf := make([]byte, 2<<27)
 289         n, err := io.ReadFull(stdout, buf)
 290         // Discard (but measure size of) anything past 128 MiB.
 291         var discarded int64
 292         if err == io.ErrUnexpectedEOF {
 293                 buf = buf[:n]
 294         } else {
 295                 c.Assert(err, check.IsNil)
 296                 discarded, err = io.Copy(ioutil.Discard, stdout)
 297                 c.Assert(err, check.IsNil)
 298         }
 299         err = cmd.Wait()
 300         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 301         // response from the server, even if the response status
 302         // indicates that the request failed. In our test suite, we
 303         // always expect a valid HTTP response, and we parse the
 304         // headers ourselves. If curl exits non-zero, our testing
 305         // environment is broken.
 306         c.Assert(err, check.Equals, nil)
 307         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 308         c.Assert(len(hdrsAndBody), check.Equals, 2)
 309         hdr = hdrsAndBody[0]
 310         bodyPart = hdrsAndBody[1]
 311         bodySize = int64(len(bodyPart)) + discarded
 312         return
 313 }
 314
 315 // Run a full-featured server, including the metrics/health routes
 316 // that are added by service.Command.
 317 func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
 318         logbuf = &bytes.Buffer{}
 319         cluster = *s.handler.Cluster
 320         cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
 321         cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
 322
 323         var configjson bytes.Buffer
 324         json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
 325         go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
 326         for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
 327                 if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
 328                         srvaddr = "http://" + m[1]
 329                         break
 330                 }
 331         }
 332         if srvaddr == "" {
 333                 c.Fatal("timed out")
 334         }
 335         return
 336 }
 337
 338 // Ensure uploads can take longer than API.RequestTimeout.
 339 //
 340 // Currently, this works only by accident: service.Command cancels the
 341 // request context as usual (there is no exemption), but
 342 // webdav.Handler doesn't notice if the request context is cancelled
 343 // while waiting to send or receive file data.
 344 func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
 345         s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
 346         _, srvaddr, _ := s.runServer(c)
 347
 348         var coll arvados.Collection
 349         arv, err := arvadosclient.MakeArvadosClient()
 350         c.Assert(err, check.IsNil)
 351         arv.ApiToken = arvadostest.ActiveTokenV2
 352         err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
 353         c.Assert(err, check.IsNil)
 354
 355         pr, pw := io.Pipe()
 356         go func() {
 357                 time.Sleep(time.Second)
 358                 pw.Write(make([]byte, 10000000))
 359                 pw.Close()
 360         }()
 361         req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
 362         req.Host = coll.UUID + ".example"
 363         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 364         resp, err := http.DefaultClient.Do(req)
 365         c.Assert(err, check.IsNil)
 366         c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
 367
 368         req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
 369         req.Host = coll.UUID + ".example"
 370         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 371         resp, err = http.DefaultClient.Do(req)
 372         c.Assert(err, check.IsNil)
 373         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 374         time.Sleep(time.Second)
 375         body, err := ioutil.ReadAll(resp.Body)
 376         c.Check(err, check.IsNil)
 377         c.Check(len(body), check.Equals, 10000000)
 378 }
 379
 380 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 381         cluster, srvaddr, _ := s.runServer(c)
 382         req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
 383         req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
 384         resp, err := http.DefaultClient.Do(req)
 385         c.Assert(err, check.IsNil)
 386         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 387         body, _ := ioutil.ReadAll(resp.Body)
 388         c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
 389 }
 390
 391 func (s *IntegrationSuite) TestMetrics(c *check.C) {
 392         cluster, srvaddr, _ := s.runServer(c)
 393
 394         req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
 395         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 396         _, err := http.DefaultClient.Do(req)
 397         c.Assert(err, check.IsNil)
 398         req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
 399         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 400         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 401         resp, err := http.DefaultClient.Do(req)
 402         c.Assert(err, check.IsNil)
 403         c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
 404         for i := 0; i < 2; i++ {
 405                 req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
 406                 req.Host = arvadostest.FooCollection + ".example.com"
 407                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 408                 resp, err = http.DefaultClient.Do(req)
 409                 c.Assert(err, check.IsNil)
 410                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 411                 buf, _ := ioutil.ReadAll(resp.Body)
 412                 c.Check(buf, check.DeepEquals, []byte("foo"))
 413                 resp.Body.Close()
 414         }
 415
 416         var coll arvados.Collection
 417         arv, err := arvadosclient.MakeArvadosClient()
 418         c.Assert(err, check.IsNil)
 419         arv.ApiToken = arvadostest.ActiveTokenV2
 420         err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
 421         c.Assert(err, check.IsNil)
 422         defer arv.Delete("collections", coll.UUID, nil, nil)
 423         for i := 0; i < 2; i++ {
 424                 size := 1 << (i * 12)
 425                 req, _ = http.NewRequest("PUT", srvaddr+"/zero-"+fmt.Sprintf("%d", size), bytes.NewReader(make([]byte, size)))
 426                 req.Host = coll.UUID + ".example.com"
 427                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 428                 resp, err = http.DefaultClient.Do(req)
 429                 c.Assert(err, check.IsNil)
 430                 c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
 431                 resp.Body.Close()
 432         }
 433
 434         time.Sleep(metricsUpdateInterval * 2)
 435
 436         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 437         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 438         resp, err = http.DefaultClient.Do(req)
 439         c.Assert(err, check.IsNil)
 440         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 441
 442         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 443         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 444         req.Header.Set("Authorization", "Bearer badtoken")
 445         resp, err = http.DefaultClient.Do(req)
 446         c.Assert(err, check.IsNil)
 447         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
 448
 449         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 450         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 451         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 452         resp, err = http.DefaultClient.Do(req)
 453         c.Assert(err, check.IsNil)
 454         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 455         type summary struct {
 456                 SampleCount string
 457                 SampleSum   float64
 458         }
 459         type counter struct {
 460                 Value int64
 461         }
 462         type gauge struct {
 463                 Value float64
 464         }
 465         var ents []struct {
 466                 Name   string
 467                 Help   string
 468                 Type   string
 469                 Metric []struct {
 470                         Label []struct {
 471                                 Name  string
 472                                 Value string
 473                         }
 474                         Counter counter
 475                         Gauge   gauge
 476                         Summary summary
 477                 }
 478         }
 479         json.NewDecoder(resp.Body).Decode(&ents)
 480         summaries := map[string]summary{}
 481         gauges := map[string]gauge{}
 482         counters := map[string]counter{}
 483         for _, e := range ents {
 484                 for _, m := range e.Metric {
 485                         labels := map[string]string{}
 486                         for _, lbl := range m.Label {
 487                                 labels[lbl.Name] = lbl.Value
 488                         }
 489                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
 490                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
 491                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 492                 }
 493         }
 494         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
 495         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 496         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 497         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
 498         c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(992))
 499
 500         // If the Host header indicates a collection, /metrics.json
 501         // refers to a file in the collection -- the metrics handler
 502         // must not intercept that route. Ditto health check paths.
 503         for _, path := range []string{"/metrics.json", "/_health/ping"} {
 504                 c.Logf("path: %q", path)
 505                 req, _ = http.NewRequest("GET", srvaddr+path, nil)
 506                 req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
 507                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 508                 resp, err = http.DefaultClient.Do(req)
 509                 c.Assert(err, check.IsNil)
 510                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 511         }
 512
 513         req, _ = http.NewRequest("GET", srvaddr+"/metrics", nil)
 514         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 515         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 516         resp, err = http.DefaultClient.Do(req)
 517         c.Assert(err, check.IsNil)
 518         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 519         allmetrics, err := ioutil.ReadAll(resp.Body)
 520         c.Check(err, check.IsNil)
 521
 522         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_apparent_backend_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
 523         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
 524         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_speed_bucket{size_range="0",le="\+Inf"} 2\n.*`)
 525         c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_sync_delay_seconds_bucket{size_range="0",le="10"} 2\n.*`)
 526
 527         c.Logf("%s", allmetrics)
 528 }
 529
 530 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 531         arvadostest.ResetDB(c)
 532         arvadostest.StartKeep(2, true)
 533
 534         arv, err := arvadosclient.MakeArvadosClient()
 535         c.Assert(err, check.Equals, nil)
 536         arv.ApiToken = arvadostest.ActiveToken
 537         kc, err := keepclient.MakeKeepClient(arv)
 538         c.Assert(err, check.Equals, nil)
 539         kc.PutB([]byte("Hello world\n"))
 540         kc.PutB([]byte("foo"))
 541         kc.PutB([]byte("foobar"))
 542         kc.PutB([]byte("waz"))
 543 }
 544
 545 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 546         arvadostest.StopKeep(2)
 547 }
 548
 549 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 550         arvadostest.ResetEnv()
 551         logger := ctxlog.TestLogger(c)
 552         ldr := config.NewLoader(&bytes.Buffer{}, logger)
 553         cfg, err := ldr.Load()
 554         c.Assert(err, check.IsNil)
 555         cluster, err := cfg.GetCluster("")
 556         c.Assert(err, check.IsNil)
 557
 558         s.ctx = ctxlog.Context(context.Background(), logger)
 559         s.handler = newHandlerOrErrorHandler(s.ctx, cluster, cluster.SystemRootToken, prometheus.NewRegistry()).(*handler)
 560         s.testServer = httptest.NewUnstartedServer(
 561                 httpserver.AddRequestIDs(
 562                         httpserver.LogRequests(
 563                                 s.handler)))
 564         s.testServer.Config.BaseContext = func(net.Listener) context.Context { return s.ctx }
 565         s.testServer.Start()
 566
 567         cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 568         cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 569 }
 570
 571 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 572         if s.testServer != nil {
 573                 s.testServer.Close()
 574         }
 575         if s.handler != nil {
 576                 s.handler.Close()
 577         }
 578 }
 579
 580 // Gocheck boilerplate
 581 func Test(t *testing.T) {
 582         check.TestingT(t)
 583 }