X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/70e5c7a3c6a5860d702d5e5c219dc0f3a3696d35..763f629e11df304e6202fb140adc27d3a08ac1a6:/services/keep-web/cache_test.go diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go index d147573eec..010e29a0b8 100644 --- a/services/keep-web/cache_test.go +++ b/services/keep-web/cache_test.go @@ -2,20 +2,25 @@ // // SPDX-License-Identifier: AGPL-3.0 -package main +package keepweb import ( "bytes" - - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "github.com/prometheus/client_golang/prometheus" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "time" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" "github.com/prometheus/common/expfmt" "gopkg.in/check.v1" ) -func (s *UnitSuite) checkCacheMetrics(c *check.C, reg *prometheus.Registry, regs ...string) { +func (s *IntegrationSuite) checkCacheMetrics(c *check.C, regs ...string) { + s.handler.Cache.updateGauges() + reg := s.handler.Cache.registry mfs, err := reg.Gather() c.Check(err, check.IsNil) buf := &bytes.Buffer{} @@ -24,128 +29,139 @@ func (s *UnitSuite) checkCacheMetrics(c *check.C, reg *prometheus.Registry, regs c.Check(enc.Encode(mf), check.IsNil) } mm := buf.String() + // Remove comments to make the "value vs. regexp" failure + // output easier to read. + mm = regexp.MustCompile(`(?m)^#.*\n`).ReplaceAllString(mm, "") for _, reg := range regs { - c.Check(mm, check.Matches, `(?ms).*collectioncache_`+reg+`\n.*`) + c.Check(mm, check.Matches, `(?ms).*keepweb_sessions_`+reg+`\n.*`) } } -func (s *UnitSuite) TestCache(c *check.C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) - - cache := DefaultConfig().Cache - cache.registry = prometheus.NewRegistry() - +func (s *IntegrationSuite) TestCache(c *check.C) { // Hit the same collection 5 times using the same token. Only // the first req should cause an API call; the next 4 should // hit all caches. - arv.ApiToken = arvadostest.AdminToken - var coll *arvados.Collection + u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } for i := 0; i < 5; i++ { - coll, err = cache.Get(arv, arvadostest.FooCollection, false) - c.Check(err, check.Equals, nil) - c.Assert(coll, check.NotNil) - c.Check(coll.PortableDataHash, check.Equals, arvadostest.FooPdh) - c.Check(coll.ManifestText[:2], check.Equals, ". ") + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) } - s.checkCacheMetrics(c, cache.registry, - "requests 5", + s.checkCacheMetrics(c, "hits 4", - "permission_hits 4", - "pdh_hits 4", - "api_calls 1") - - // Hit the same collection 2 more times, this time requesting - // it by PDH and using a different token. The first req should - // miss the permission cache and fetch the new manifest; the - // second should hit the Collection cache and skip the API - // lookup. - arv.ApiToken = arvadostest.ActiveToken - - coll2, err := cache.Get(arv, arvadostest.FooPdh, false) - c.Check(err, check.Equals, nil) - c.Assert(coll2, check.NotNil) - c.Check(coll2.PortableDataHash, check.Equals, arvadostest.FooPdh) - c.Check(coll2.ManifestText[:2], check.Equals, ". ") - c.Check(coll2.ManifestText, check.Not(check.Equals), coll.ManifestText) - - s.checkCacheMetrics(c, cache.registry, - "requests 6", - "hits 4", - "permission_hits 4", - "pdh_hits 4", - "api_calls 2") - - coll2, err = cache.Get(arv, arvadostest.FooPdh, false) - c.Check(err, check.Equals, nil) - c.Assert(coll2, check.NotNil) - c.Check(coll2.PortableDataHash, check.Equals, arvadostest.FooPdh) - c.Check(coll2.ManifestText[:2], check.Equals, ". ") - - s.checkCacheMetrics(c, cache.registry, - "requests 7", - "hits 5", - "permission_hits 5", - "pdh_hits 4", - "api_calls 2") - - // Alternating between two collections N times should produce - // only 2 more API calls. - arv.ApiToken = arvadostest.AdminToken - for i := 0; i < 20; i++ { - var target string - if i%2 == 0 { - target = arvadostest.HelloWorldCollection - } else { - target = arvadostest.FooBarDirCollection - } - _, err := cache.Get(arv, target, false) - c.Check(err, check.Equals, nil) + "misses 1", + "active 1") + + // Hit a shared collection 3 times using PDH, using a + // different token. + u2 := mustParseURL("http://" + strings.Replace(arvadostest.BarFileCollectionPDH, "+", "-", 1) + ".keep-web.example/bar") + req2 := &http.Request{ + Method: "GET", + Host: u2.Host, + URL: u2, + RequestURI: u2.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.SpectatorToken}, + }, } - s.checkCacheMetrics(c, cache.registry, - "requests 27", - "hits 23", - "permission_hits 23", - "pdh_hits 22", - "api_calls 4") -} - -func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) - - cache := DefaultConfig().Cache - cache.registry = prometheus.NewRegistry() - - for _, forceReload := range []bool{false, true, false, true} { - _, err := cache.Get(arv, arvadostest.FooPdh, forceReload) - c.Check(err, check.Equals, nil) + for i := 0; i < 3; i++ { + resp2 := httptest.NewRecorder() + s.handler.ServeHTTP(resp2, req2) + c.Check(resp2.Code, check.Equals, http.StatusOK) } - - s.checkCacheMetrics(c, cache.registry, - "requests 4", - "hits 3", - "permission_hits 1", - "pdh_hits 0", - "api_calls 3") + s.checkCacheMetrics(c, + "hits 6", + "misses 2", + "active 2") + + // Alternating between two collections/tokens N times should + // use the existing sessions. + for i := 0; i < 7; i++ { + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + + resp2 := httptest.NewRecorder() + s.handler.ServeHTTP(resp2, req2) + c.Check(resp2.Code, check.Equals, http.StatusOK) + } + s.checkCacheMetrics(c, + "hits 20", + "misses 2", + "active 2") } -func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) - - cache := DefaultConfig().Cache - cache.registry = prometheus.NewRegistry() - - for _, forceReload := range []bool{false, true, false, true} { - _, err := cache.Get(arv, arvadostest.FooCollection, forceReload) - c.Check(err, check.Equals, nil) - } +func (s *IntegrationSuite) TestForceReloadPDH(c *check.C) { + filename := strings.Replace(time.Now().Format(time.RFC3339Nano), ":", ".", -1) + manifest := ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:" + filename + "\n" + pdh := arvados.PortableDataHash(manifest) + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.ActiveToken + + _, resp := s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/"+filename, arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + + var coll arvados.Collection + err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": manifest, + }, + }) + c.Assert(err, check.IsNil) + defer client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) + c.Assert(coll.PortableDataHash, check.Equals, pdh) + + _, resp = s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/"+filename, "", http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + "Cache-Control": {"must-revalidate"}, + }) + c.Check(resp.Code, check.Equals, http.StatusOK) + + _, resp = s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/missingfile", "", http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + "Cache-Control": {"must-revalidate"}, + }) + c.Check(resp.Code, check.Equals, http.StatusNotFound) +} - s.checkCacheMetrics(c, cache.registry, - "requests 4", - "hits 3", - "permission_hits 1", - "pdh_hits 3", - "api_calls 3") +func (s *IntegrationSuite) TestForceReloadUUID(c *check.C) { + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.ActiveToken + var coll arvados.Collection + err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:oldfile\n", + }, + }) + c.Assert(err, check.IsNil) + defer client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) + + _, resp := s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/oldfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusOK) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + err = client.RequestAndDecode(&coll, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:oldfile 0:0:newfile\n", + }, + }) + c.Assert(err, check.IsNil) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", "", http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + "Cache-Control": {"must-revalidate"}, + }) + c.Check(resp.Code, check.Equals, http.StatusOK) }