services/keep-web/cache_test.go

   1 // Copyright (C) The Arvados Authors. All rights reserved.
   2 //
   3 // SPDX-License-Identifier: AGPL-3.0
   4
   5 package keepweb
   6
   7 import (
   8         "bytes"
   9         "net/http"
  10         "net/http/httptest"
  11         "regexp"
  12         "strings"
  13         "time"
  14
  15         "git.arvados.org/arvados.git/sdk/go/arvados"
  16         "git.arvados.org/arvados.git/sdk/go/arvadostest"
  17         "github.com/prometheus/common/expfmt"
  18         "gopkg.in/check.v1"
  19 )
  20
  21 func (s *IntegrationSuite) checkCacheMetrics(c *check.C, regs ...string) {
  22         s.handler.Cache.updateGauges()
  23         reg := s.handler.Cache.registry
  24         mfs, err := reg.Gather()
  25         c.Check(err, check.IsNil)
  26         buf := &bytes.Buffer{}
  27         enc := expfmt.NewEncoder(buf, expfmt.FmtText)
  28         for _, mf := range mfs {
  29                 c.Check(enc.Encode(mf), check.IsNil)
  30         }
  31         mm := buf.String()
  32         // Remove comments to make the "value vs. regexp" failure
  33         // output easier to read.
  34         mm = regexp.MustCompile(`(?m)^#.*\n`).ReplaceAllString(mm, "")
  35         for _, reg := range regs {
  36                 c.Check(mm, check.Matches, `(?ms).*keepweb_sessions_`+reg+`\n.*`)
  37         }
  38 }
  39
  40 func (s *IntegrationSuite) TestCache(c *check.C) {
  41         // Hit the same collection 5 times using the same token. Only
  42         // the first req should cause an API call; the next 4 should
  43         // hit all caches.
  44         u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo")
  45         req := &http.Request{
  46                 Method:     "GET",
  47                 Host:       u.Host,
  48                 URL:        u,
  49                 RequestURI: u.RequestURI(),
  50                 Header: http.Header{
  51                         "Authorization": {"Bearer " + arvadostest.ActiveToken},
  52                 },
  53         }
  54         for i := 0; i < 5; i++ {
  55                 resp := httptest.NewRecorder()
  56                 s.handler.ServeHTTP(resp, req)
  57                 c.Check(resp.Code, check.Equals, http.StatusOK)
  58         }
  59         s.checkCacheMetrics(c,
  60                 "hits 4",
  61                 "misses 1",
  62                 "active 1")
  63
  64         // Hit a shared collection 3 times using PDH, using a
  65         // different token.
  66         u2 := mustParseURL("http://" + strings.Replace(arvadostest.BarFileCollectionPDH, "+", "-", 1) + ".keep-web.example/bar")
  67         req2 := &http.Request{
  68                 Method:     "GET",
  69                 Host:       u2.Host,
  70                 URL:        u2,
  71                 RequestURI: u2.RequestURI(),
  72                 Header: http.Header{
  73                         "Authorization": {"Bearer " + arvadostest.SpectatorToken},
  74                 },
  75         }
  76         for i := 0; i < 3; i++ {
  77                 resp2 := httptest.NewRecorder()
  78                 s.handler.ServeHTTP(resp2, req2)
  79                 c.Check(resp2.Code, check.Equals, http.StatusOK)
  80         }
  81         s.checkCacheMetrics(c,
  82                 "hits 6",
  83                 "misses 2",
  84                 "active 2")
  85
  86         // Alternating between two collections/tokens N times should
  87         // use the existing sessions.
  88         for i := 0; i < 7; i++ {
  89                 resp := httptest.NewRecorder()
  90                 s.handler.ServeHTTP(resp, req)
  91                 c.Check(resp.Code, check.Equals, http.StatusOK)
  92
  93                 resp2 := httptest.NewRecorder()
  94                 s.handler.ServeHTTP(resp2, req2)
  95                 c.Check(resp2.Code, check.Equals, http.StatusOK)
  96         }
  97         s.checkCacheMetrics(c,
  98                 "hits 20",
  99                 "misses 2",
 100                 "active 2")
 101 }
 102
 103 func (s *IntegrationSuite) TestForceReloadPDH(c *check.C) {
 104         filename := strings.Replace(time.Now().Format(time.RFC3339Nano), ":", ".", -1)
 105         manifest := ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:" + filename + "\n"
 106         pdh := arvados.PortableDataHash(manifest)
 107         client := arvados.NewClientFromEnv()
 108         client.AuthToken = arvadostest.ActiveToken
 109
 110         _, resp := s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/"+filename, arvadostest.ActiveToken, nil)
 111         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 112
 113         var coll arvados.Collection
 114         err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 115                 "collection": map[string]string{
 116                         "manifest_text": manifest,
 117                 },
 118         })
 119         c.Assert(err, check.IsNil)
 120         defer client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 121         c.Assert(coll.PortableDataHash, check.Equals, pdh)
 122
 123         _, resp = s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/"+filename, "", http.Header{
 124                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 125                 "Cache-Control": {"must-revalidate"},
 126         })
 127         c.Check(resp.Code, check.Equals, http.StatusOK)
 128
 129         _, resp = s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/missingfile", "", http.Header{
 130                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 131                 "Cache-Control": {"must-revalidate"},
 132         })
 133         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 134 }
 135
 136 func (s *IntegrationSuite) TestForceReloadUUID(c *check.C) {
 137         client := arvados.NewClientFromEnv()
 138         client.AuthToken = arvadostest.ActiveToken
 139         var coll arvados.Collection
 140         err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
 141                 "collection": map[string]string{
 142                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:oldfile\n",
 143                 },
 144         })
 145         c.Assert(err, check.IsNil)
 146         defer client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil)
 147
 148         _, resp := s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil)
 149         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 150         _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/oldfile", arvadostest.ActiveToken, nil)
 151         c.Check(resp.Code, check.Equals, http.StatusOK)
 152         _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil)
 153         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 154         err = client.RequestAndDecode(&coll, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{
 155                 "collection": map[string]string{
 156                         "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:oldfile 0:0:newfile\n",
 157                 },
 158         })
 159         c.Assert(err, check.IsNil)
 160         _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil)
 161         c.Check(resp.Code, check.Equals, http.StatusNotFound)
 162         _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", "", http.Header{
 163                 "Authorization": {"Bearer " + arvadostest.ActiveToken},
 164                 "Cache-Control": {"must-revalidate"},
 165         })
 166         c.Check(resp.Code, check.Equals, http.StatusOK)
 167 }