services/keep-web/server_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "context"
  10         "crypto/md5"
  11         "encoding/json"
  12         "fmt"
  13         "io"
  14         "io/ioutil"
  15         "net"
  16         "net/http"
  17         "net/http/httptest"
  18         "os"
  19         "os/exec"
  20         "regexp"
  21         "strings"
  22         "testing"
  23         "time"
  24
  25         "git.arvados.org/arvados.git/lib/config"
  26         "git.arvados.org/arvados.git/sdk/go/arvados"
  27         "git.arvados.org/arvados.git/sdk/go/arvadosclient"
  28         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  29         "git.arvados.org/arvados.git/sdk/go/ctxlog"
  30         "git.arvados.org/arvados.git/sdk/go/httpserver"
  31         "git.arvados.org/arvados.git/sdk/go/keepclient"
  32         check "gopkg.in/check.v1"
  33 )
  34
  35 var testAPIHost = os.Getenv("ARVADOS_API_HOST")
  36
  37 var _ = check.Suite(&IntegrationSuite{})
  38
  39 // IntegrationSuite tests need an API server and a keep-web server
  40 type IntegrationSuite struct {
  41         testServer *httptest.Server
  42         handler    *handler
  43 }
  44
  45 func (s *IntegrationSuite) TestNoToken(c *check.C) {
  46         for _, token := range []string{
  47                 "",
  48                 "bogustoken",
  49         } {
  50                 hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
  51                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  52                 c.Check(body, check.Equals, notFoundMessage+"\n")
  53
  54                 if token != "" {
  55                         hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
  56                         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  57                         c.Check(body, check.Equals, notFoundMessage+"\n")
  58                 }
  59
  60                 hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
  61                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
  62                 c.Check(body, check.Equals, notFoundMessage+"\n")
  63         }
  64 }
  65
  66 // TODO: Move most cases to functional tests -- at least use Go's own
  67 // http client instead of forking curl. Just leave enough of an
  68 // integration test to assure that the documented way of invoking curl
  69 // really works against the server.
  70 func (s *IntegrationSuite) Test404(c *check.C) {
  71         for _, uri := range []string{
  72                 // Routing errors (always 404 regardless of what's stored in Keep)
  73                 "/foo",
  74                 "/download",
  75                 "/collections",
  76                 "/collections/",
  77                 // Implicit/generated index is not implemented yet;
  78                 // until then, return 404.
  79                 "/collections/" + arvadostest.FooCollection,
  80                 "/collections/" + arvadostest.FooCollection + "/",
  81                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1",
  82                 "/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
  83                 // Non-existent file in collection
  84                 "/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
  85                 "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  86                 // Non-existent collection
  87                 "/collections/" + arvadostest.NonexistentCollection,
  88                 "/collections/" + arvadostest.NonexistentCollection + "/",
  89                 "/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
  90                 "/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
  91         } {
  92                 hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
  93                 c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
  94                 if len(body) > 0 {
  95                         c.Check(body, check.Equals, notFoundMessage+"\n")
  96                 }
  97         }
  98 }
  99
 100 func (s *IntegrationSuite) Test1GBFile(c *check.C) {
 101         if testing.Short() {
 102                 c.Skip("skipping 1GB integration test in short mode")
 103         }
 104         s.test100BlockFile(c, 10000000)
 105 }
 106
 107 func (s *IntegrationSuite) Test100BlockFile(c *check.C) {
 108         if testing.Short() {
 109                 // 3 MB
 110                 s.test100BlockFile(c, 30000)
 111         } else {
 112                 // 300 MB
 113                 s.test100BlockFile(c, 3000000)
 114         }
 115 }
 116
 117 func (s *IntegrationSuite) test100BlockFile(c *check.C, blocksize int) {
 118         testdata := make([]byte, blocksize)
 119         for i := 0; i < blocksize; i++ {
 120                 testdata[i] = byte(' ')
 121         }
 122         arv, err := arvadosclient.MakeArvadosClient()
 123         c.Assert(err, check.Equals, nil)
 124         arv.ApiToken = arvadostest.ActiveToken
 125         kc, err := keepclient.MakeKeepClient(arv)
 126         c.Assert(err, check.Equals, nil)
 127         loc, _, err := kc.PutB(testdata[:])
 128         c.Assert(err, check.Equals, nil)
 129         mtext := "."
 130         for i := 0; i < 100; i++ {
 131                 mtext = mtext + " " + loc
 132         }
 133         mtext = mtext + fmt.Sprintf(" 0:%d00:testdata.bin\n", blocksize)
 134         coll := map[string]interface{}{}
 135         err = arv.Create("collections",
 136                 map[string]interface{}{
 137                         "collection": map[string]interface{}{
 138                                 "name":          fmt.Sprintf("testdata blocksize=%d", blocksize),
 139                                 "manifest_text": mtext,
 140                         },
 141                 }, &coll)
 142         c.Assert(err, check.Equals, nil)
 143         uuid := coll["uuid"].(string)
 144
 145         hdr, body, size := s.runCurl(c, arv.ApiToken, uuid+".collections.example.com", "/testdata.bin")
 146         c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 147         c.Check(hdr, check.Matches, `(?si).*Content-length: `+fmt.Sprintf("%d00", blocksize)+`\r\n.*`)
 148         c.Check([]byte(body)[:1234], check.DeepEquals, testdata[:1234])
 149         c.Check(size, check.Equals, int64(blocksize)*100)
 150 }
 151
 152 type curlCase struct {
 153         auth    string
 154         host    string
 155         path    string
 156         dataMD5 string
 157 }
 158
 159 func (s *IntegrationSuite) Test200(c *check.C) {
 160         s.handler.Cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken
 161         for _, spec := range []curlCase{
 162                 // My collection
 163                 {
 164                         auth:    arvadostest.ActiveToken,
 165                         host:    arvadostest.FooCollection + "--collections.example.com",
 166                         path:    "/foo",
 167                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 168                 },
 169                 {
 170                         auth:    arvadostest.ActiveToken,
 171                         host:    arvadostest.FooCollection + ".collections.example.com",
 172                         path:    "/foo",
 173                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 174                 },
 175                 {
 176                         host:    strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com",
 177                         path:    "/t=" + arvadostest.ActiveToken + "/foo",
 178                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 179                 },
 180                 {
 181                         path:    "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo",
 182                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 183                 },
 184                 {
 185                         path:    "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo",
 186                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 187                 },
 188                 {
 189                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 190                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 191                 },
 192                 {
 193                         auth:    "tokensobogus",
 194                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 195                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 196                 },
 197                 {
 198                         auth:    arvadostest.ActiveToken,
 199                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 200                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 201                 },
 202                 {
 203                         auth:    arvadostest.AnonymousToken,
 204                         path:    "/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/foo",
 205                         dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8",
 206                 },
 207
 208                 // Anonymously accessible data
 209                 {
 210                         path:    "/c=" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 211                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 212                 },
 213                 {
 214                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 215                         path:    "/Hello%20world.txt",
 216                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 217                 },
 218                 {
 219                         host:    arvadostest.HelloWorldCollection + ".collections.example.com",
 220                         path:    "/_/Hello%20world.txt",
 221                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 222                 },
 223                 {
 224                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 225                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 226                 },
 227                 {
 228                         auth:    arvadostest.ActiveToken,
 229                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 230                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 231                 },
 232                 {
 233                         auth:    arvadostest.SpectatorToken,
 234                         path:    "/collections/" + arvadostest.HelloWorldCollection + "/Hello%20world.txt",
 235                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 236                 },
 237                 {
 238                         auth:    arvadostest.SpectatorToken,
 239                         host:    arvadostest.HelloWorldCollection + "--collections.example.com",
 240                         path:    "/Hello%20world.txt",
 241                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 242                 },
 243                 {
 244                         auth:    arvadostest.SpectatorToken,
 245                         path:    "/collections/download/" + arvadostest.HelloWorldCollection + "/" + arvadostest.SpectatorToken + "/Hello%20world.txt",
 246                         dataMD5: "f0ef7081e1539ac00ef5b761b4fb01b3",
 247                 },
 248         } {
 249                 host := spec.host
 250                 if host == "" {
 251                         host = "collections.example.com"
 252                 }
 253                 hdr, body, _ := s.runCurl(c, spec.auth, host, spec.path)
 254                 c.Check(hdr, check.Matches, `(?s)HTTP/1.1 200 OK\r\n.*`)
 255                 if strings.HasSuffix(spec.path, ".txt") {
 256                         c.Check(hdr, check.Matches, `(?s).*\r\nContent-Type: text/plain.*`)
 257                         // TODO: Check some types that aren't
 258                         // automatically detected by Go's http server
 259                         // by sniffing the content.
 260                 }
 261                 c.Check(fmt.Sprintf("%x", md5.Sum([]byte(body))), check.Equals, spec.dataMD5)
 262         }
 263 }
 264
 265 // Return header block and body.
 266 func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 267         curlArgs := []string{"--silent", "--show-error", "--include"}
 268         testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
 269         curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
 270         if strings.Contains(auth, " ") {
 271                 // caller supplied entire Authorization header value
 272                 curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
 273         } else if auth != "" {
 274                 // caller supplied Arvados token
 275                 curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 276         }
 277         curlArgs = append(curlArgs, args...)
 278         curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
 279         c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 280         cmd := exec.Command("curl", curlArgs...)
 281         stdout, err := cmd.StdoutPipe()
 282         c.Assert(err, check.IsNil)
 283         cmd.Stderr = os.Stderr
 284         err = cmd.Start()
 285         c.Assert(err, check.IsNil)
 286         buf := make([]byte, 2<<27)
 287         n, err := io.ReadFull(stdout, buf)
 288         // Discard (but measure size of) anything past 128 MiB.
 289         var discarded int64
 290         if err == io.ErrUnexpectedEOF {
 291                 buf = buf[:n]
 292         } else {
 293                 c.Assert(err, check.IsNil)
 294                 discarded, err = io.Copy(ioutil.Discard, stdout)
 295                 c.Assert(err, check.IsNil)
 296         }
 297         err = cmd.Wait()
 298         // Without "-f", curl exits 0 as long as it gets a valid HTTP
 299         // response from the server, even if the response status
 300         // indicates that the request failed. In our test suite, we
 301         // always expect a valid HTTP response, and we parse the
 302         // headers ourselves. If curl exits non-zero, our testing
 303         // environment is broken.
 304         c.Assert(err, check.Equals, nil)
 305         hdrsAndBody := strings.SplitN(string(buf), "\r\n\r\n", 2)
 306         c.Assert(len(hdrsAndBody), check.Equals, 2)
 307         hdr = hdrsAndBody[0]
 308         bodyPart = hdrsAndBody[1]
 309         bodySize = int64(len(bodyPart)) + discarded
 310         return
 311 }
 312
 313 // Run a full-featured server, including the metrics/health routes
 314 // that are added by service.Command.
 315 func (s *IntegrationSuite) runServer(c *check.C) (cluster arvados.Cluster, srvaddr string, logbuf *bytes.Buffer) {
 316         logbuf = &bytes.Buffer{}
 317         cluster = *s.handler.Cluster
 318         cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
 319         cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Scheme: "http", Host: "0.0.0.0:0"}: {}}
 320
 321         var configjson bytes.Buffer
 322         json.NewEncoder(&configjson).Encode(arvados.Config{Clusters: map[string]arvados.Cluster{"zzzzz": cluster}})
 323         go Command.RunCommand("keep-web", []string{"-config=-"}, &configjson, os.Stderr, io.MultiWriter(os.Stderr, logbuf))
 324         for deadline := time.Now().Add(time.Second); deadline.After(time.Now()); time.Sleep(time.Second / 100) {
 325                 if m := regexp.MustCompile(`"Listen":"(.*?)"`).FindStringSubmatch(logbuf.String()); m != nil {
 326                         srvaddr = "http://" + m[1]
 327                         break
 328                 }
 329         }
 330         if srvaddr == "" {
 331                 c.Fatal("timed out")
 332         }
 333         return
 334 }
 335
 336 // Ensure uploads can take longer than API.RequestTimeout.
 337 //
 338 // Currently, this works only by accident: service.Command cancels the
 339 // request context as usual (there is no exemption), but
 340 // webdav.Handler doesn't notice if the request context is cancelled
 341 // while waiting to send or receive file data.
 342 func (s *IntegrationSuite) TestRequestTimeoutExemption(c *check.C) {
 343         s.handler.Cluster.API.RequestTimeout = arvados.Duration(time.Second / 2)
 344         _, srvaddr, _ := s.runServer(c)
 345
 346         var coll arvados.Collection
 347         arv, err := arvadosclient.MakeArvadosClient()
 348         c.Assert(err, check.IsNil)
 349         arv.ApiToken = arvadostest.ActiveTokenV2
 350         err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
 351         c.Assert(err, check.IsNil)
 352
 353         pr, pw := io.Pipe()
 354         go func() {
 355                 time.Sleep(time.Second)
 356                 pw.Write(make([]byte, 10000000))
 357                 pw.Close()
 358         }()
 359         req, _ := http.NewRequest("PUT", srvaddr+"/testfile", pr)
 360         req.Host = coll.UUID + ".example"
 361         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 362         resp, err := http.DefaultClient.Do(req)
 363         c.Assert(err, check.IsNil)
 364         c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
 365
 366         req, _ = http.NewRequest("GET", srvaddr+"/testfile", nil)
 367         req.Host = coll.UUID + ".example"
 368         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveTokenV2)
 369         resp, err = http.DefaultClient.Do(req)
 370         c.Assert(err, check.IsNil)
 371         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 372         time.Sleep(time.Second)
 373         body, err := ioutil.ReadAll(resp.Body)
 374         c.Check(err, check.IsNil)
 375         c.Check(len(body), check.Equals, 10000000)
 376 }
 377
 378 func (s *IntegrationSuite) TestHealthCheckPing(c *check.C) {
 379         cluster, srvaddr, _ := s.runServer(c)
 380         req, _ := http.NewRequest("GET", srvaddr+"/_health/ping", nil)
 381         req.Header.Set("Authorization", "Bearer "+cluster.ManagementToken)
 382         resp, err := http.DefaultClient.Do(req)
 383         c.Assert(err, check.IsNil)
 384         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 385         body, _ := ioutil.ReadAll(resp.Body)
 386         c.Check(string(body), check.Matches, `{"health":"OK"}\n`)
 387 }
 388
 389 func (s *IntegrationSuite) TestMetrics(c *check.C) {
 390         cluster, srvaddr, _ := s.runServer(c)
 391
 392         req, _ := http.NewRequest("GET", srvaddr+"/notfound", nil)
 393         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 394         _, err := http.DefaultClient.Do(req)
 395         c.Assert(err, check.IsNil)
 396         req, _ = http.NewRequest("GET", srvaddr+"/by_id/", nil)
 397         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 398         req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 399         resp, err := http.DefaultClient.Do(req)
 400         c.Assert(err, check.IsNil)
 401         c.Assert(resp.StatusCode, check.Equals, http.StatusOK)
 402         for i := 0; i < 2; i++ {
 403                 req, _ = http.NewRequest("GET", srvaddr+"/foo", nil)
 404                 req.Host = arvadostest.FooCollection + ".example.com"
 405                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 406                 resp, err = http.DefaultClient.Do(req)
 407                 c.Assert(err, check.IsNil)
 408                 c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 409                 buf, _ := ioutil.ReadAll(resp.Body)
 410                 c.Check(buf, check.DeepEquals, []byte("foo"))
 411                 resp.Body.Close()
 412         }
 413
 414         time.Sleep(metricsUpdateInterval * 2)
 415
 416         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 417         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 418         resp, err = http.DefaultClient.Do(req)
 419         c.Assert(err, check.IsNil)
 420         c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized)
 421
 422         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 423         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 424         req.Header.Set("Authorization", "Bearer badtoken")
 425         resp, err = http.DefaultClient.Do(req)
 426         c.Assert(err, check.IsNil)
 427         c.Check(resp.StatusCode, check.Equals, http.StatusForbidden)
 428
 429         req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
 430         req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
 431         req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
 432         resp, err = http.DefaultClient.Do(req)
 433         c.Assert(err, check.IsNil)
 434         c.Check(resp.StatusCode, check.Equals, http.StatusOK)
 435         type summary struct {
 436                 SampleCount string
 437                 SampleSum   float64
 438         }
 439         type counter struct {
 440                 Value int64
 441         }
 442         type gauge struct {
 443                 Value float64
 444         }
 445         var ents []struct {
 446                 Name   string
 447                 Help   string
 448                 Type   string
 449                 Metric []struct {
 450                         Label []struct {
 451                                 Name  string
 452                                 Value string
 453                         }
 454                         Counter counter
 455                         Gauge   gauge
 456                         Summary summary
 457                 }
 458         }
 459         json.NewDecoder(resp.Body).Decode(&ents)
 460         summaries := map[string]summary{}
 461         gauges := map[string]gauge{}
 462         counters := map[string]counter{}
 463         for _, e := range ents {
 464                 for _, m := range e.Metric {
 465                         labels := map[string]string{}
 466                         for _, lbl := range m.Label {
 467                                 labels[lbl.Name] = lbl.Value
 468                         }
 469                         summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary
 470                         counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter
 471                         gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge
 472                 }
 473         }
 474         c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0)
 475         c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 476         c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 477         c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
 478         c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
 479         c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
 480         c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
 481         c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
 482         c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
 483         // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
 484         c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
 485
 486         // If the Host header indicates a collection, /metrics.json
 487         // refers to a file in the collection -- the metrics handler
 488         // must not intercept that route. Ditto health check paths.
 489         for _, path := range []string{"/metrics.json", "/_health/ping"} {
 490                 c.Logf("path: %q", path)
 491                 req, _ = http.NewRequest("GET", srvaddr+path, nil)
 492                 req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com"
 493                 req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
 494                 resp, err = http.DefaultClient.Do(req)
 495                 c.Assert(err, check.IsNil)
 496                 c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 497         }
 498 }
 499
 500 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 501         arvadostest.ResetDB(c)
 502         arvadostest.StartKeep(2, true)
 503
 504         arv, err := arvadosclient.MakeArvadosClient()
 505         c.Assert(err, check.Equals, nil)
 506         arv.ApiToken = arvadostest.ActiveToken
 507         kc, err := keepclient.MakeKeepClient(arv)
 508         c.Assert(err, check.Equals, nil)
 509         kc.PutB([]byte("Hello world\n"))
 510         kc.PutB([]byte("foo"))
 511         kc.PutB([]byte("foobar"))
 512         kc.PutB([]byte("waz"))
 513 }
 514
 515 func (s *IntegrationSuite) TearDownSuite(c *check.C) {
 516         arvadostest.StopKeep(2)
 517 }
 518
 519 func (s *IntegrationSuite) SetUpTest(c *check.C) {
 520         arvadostest.ResetEnv()
 521         logger := ctxlog.TestLogger(c)
 522         ldr := config.NewLoader(&bytes.Buffer{}, logger)
 523         cfg, err := ldr.Load()
 524         c.Assert(err, check.IsNil)
 525         cluster, err := cfg.GetCluster("")
 526         c.Assert(err, check.IsNil)
 527
 528         ctx := ctxlog.Context(context.Background(), logger)
 529
 530         s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*handler)
 531         s.testServer = httptest.NewUnstartedServer(
 532                 httpserver.AddRequestIDs(
 533                         httpserver.LogRequests(
 534                                 s.handler)))
 535         s.testServer.Config.BaseContext = func(net.Listener) context.Context { return ctx }
 536         s.testServer.Start()
 537
 538         cluster.Services.WebDAV.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 539         cluster.Services.WebDAVDownload.InternalURLs = map[arvados.URL]arvados.ServiceInstance{{Host: s.testServer.URL[7:]}: {}}
 540 }
 541
 542 func (s *IntegrationSuite) TearDownTest(c *check.C) {
 543         if s.testServer != nil {
 544                 s.testServer.Close()
 545         }
 546 }
 547
 548 // Gocheck boilerplate
 549 func Test(t *testing.T) {
 550         check.TestingT(t)
 551 }