X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3b3c3a0869c2cf528b9e1c45c969bbbd47f6446e..1a61611cbb8ed4471eb2e04e60bad548bff94678:/services/keep-web/cache_test.go diff --git a/services/keep-web/cache_test.go b/services/keep-web/cache_test.go index f8aa2b1c60..e95ebcf846 100644 --- a/services/keep-web/cache_test.go +++ b/services/keep-web/cache_test.go @@ -1,104 +1,157 @@ -package main +// Copyright (C) The Arvados Authors. All rights reserved. +// +// SPDX-License-Identifier: AGPL-3.0 + +package keepweb import ( - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" + "net/http" + "net/http/httptest" + "regexp" + "strings" + "time" + + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadostest" "gopkg.in/check.v1" ) -func (s *UnitSuite) TestCache(c *check.C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) - - cache := DefaultConfig().Cache +func (s *IntegrationSuite) checkCacheMetrics(c *check.C, regs ...string) { + s.handler.Cache.updateGauges() + mm := arvadostest.GatherMetricsAsString(s.handler.Cache.registry) + // Remove comments to make the "value vs. regexp" failure + // output easier to read. + mm = regexp.MustCompile(`(?m)^#.*\n`).ReplaceAllString(mm, "") + for _, reg := range regs { + c.Check(mm, check.Matches, `(?ms).*keepweb_sessions_`+reg+`\n.*`) + } +} +func (s *IntegrationSuite) TestCache(c *check.C) { // Hit the same collection 5 times using the same token. Only // the first req should cause an API call; the next 4 should // hit all caches. - arv.ApiToken = arvadostest.AdminToken + u := mustParseURL("http://" + arvadostest.FooCollection + ".keep-web.example/foo") + req := &http.Request{ + Method: "GET", + Host: u.Host, + URL: u, + RequestURI: u.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + }, + } for i := 0; i < 5; i++ { - coll, err := cache.Get(arv, arvadostest.FooCollection, false) - c.Check(err, check.Equals, nil) - c.Assert(coll, check.NotNil) - c.Check(coll["portable_data_hash"], check.Equals, arvadostest.FooPdh) - c.Check(coll["manifest_text"].(string)[:2], check.Equals, ". ") + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) } - c.Check(cache.Stats().Requests, check.Equals, uint64(5)) - c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4)) - c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4)) - c.Check(cache.Stats().PDHHits, check.Equals, uint64(4)) - c.Check(cache.Stats().APICalls, check.Equals, uint64(1)) + s.checkCacheMetrics(c, + "hits 4", + "misses 1", + "active 1") - // Hit the same collection 2 more times, this time requesting - // it by PDH and using a different token. The first req should - // miss the permission cache. Both reqs should hit the - // Collection cache and skip the API lookup. - arv.ApiToken = arvadostest.ActiveToken - for i := 0; i < 2; i++ { - coll, err := cache.Get(arv, arvadostest.FooPdh, false) - c.Check(err, check.Equals, nil) - c.Assert(coll, check.NotNil) - c.Check(coll["portable_data_hash"], check.Equals, arvadostest.FooPdh) - c.Check(coll["manifest_text"].(string)[:2], check.Equals, ". ") + // Hit a shared collection 3 times using PDH, using a + // different token. + u2 := mustParseURL("http://" + strings.Replace(arvadostest.BarFileCollectionPDH, "+", "-", 1) + ".keep-web.example/bar") + req2 := &http.Request{ + Method: "GET", + Host: u2.Host, + URL: u2, + RequestURI: u2.RequestURI(), + Header: http.Header{ + "Authorization": {"Bearer " + arvadostest.SpectatorToken}, + }, + } + for i := 0; i < 3; i++ { + resp2 := httptest.NewRecorder() + s.handler.ServeHTTP(resp2, req2) + c.Check(resp2.Code, check.Equals, http.StatusOK) } - c.Check(cache.Stats().Requests, check.Equals, uint64(5+2)) - c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+2)) - c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+1)) - c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0)) - c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1)) + s.checkCacheMetrics(c, + "hits 6", + "misses 2", + "active 2") - // Alternating between two collections N times should produce - // only 2 more API calls. - arv.ApiToken = arvadostest.AdminToken - for i := 0; i < 20; i++ { - var target string - if i%2 == 0 { - target = arvadostest.HelloWorldCollection - } else { - target = arvadostest.FooBarDirCollection - } - _, err := cache.Get(arv, target, false) - c.Check(err, check.Equals, nil) + // Alternating between two collections/tokens N times should + // use the existing sessions. + for i := 0; i < 7; i++ { + resp := httptest.NewRecorder() + s.handler.ServeHTTP(resp, req) + c.Check(resp.Code, check.Equals, http.StatusOK) + + resp2 := httptest.NewRecorder() + s.handler.ServeHTTP(resp2, req2) + c.Check(resp2.Code, check.Equals, http.StatusOK) } - c.Check(cache.Stats().Requests, check.Equals, uint64(5+2+20)) - c.Check(cache.Stats().CollectionHits, check.Equals, uint64(4+2+18)) - c.Check(cache.Stats().PermissionHits, check.Equals, uint64(4+1+18)) - c.Check(cache.Stats().PDHHits, check.Equals, uint64(4+0+18)) - c.Check(cache.Stats().APICalls, check.Equals, uint64(1+1+2)) + s.checkCacheMetrics(c, + "hits 20", + "misses 2", + "active 2") } -func (s *UnitSuite) TestCacheForceReloadByPDH(c *check.C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) - - cache := DefaultConfig().Cache +func (s *IntegrationSuite) TestForceReloadPDH(c *check.C) { + filename := strings.Replace(time.Now().Format(time.RFC3339Nano), ":", ".", -1) + manifest := ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:" + filename + "\n" + pdh := arvados.PortableDataHash(manifest) + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.ActiveToken - for _, forceReload := range []bool{false, true, false, true} { - _, err := cache.Get(arv, arvadostest.FooPdh, forceReload) - c.Check(err, check.Equals, nil) - } + _, resp := s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/"+filename, arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) - c.Check(cache.Stats().Requests, check.Equals, uint64(4)) - c.Check(cache.Stats().CollectionHits, check.Equals, uint64(3)) - c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1)) - c.Check(cache.Stats().PDHHits, check.Equals, uint64(0)) - c.Check(cache.Stats().APICalls, check.Equals, uint64(3)) -} + var coll arvados.Collection + err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": manifest, + }, + }) + c.Assert(err, check.IsNil) + defer client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) + c.Assert(coll.PortableDataHash, check.Equals, pdh) -func (s *UnitSuite) TestCacheForceReloadByUUID(c *check.C) { - arv, err := arvadosclient.MakeArvadosClient() - c.Assert(err, check.Equals, nil) + _, resp = s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/"+filename, "", http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + "Cache-Control": {"must-revalidate"}, + }) + c.Check(resp.Code, check.Equals, http.StatusOK) - cache := DefaultConfig().Cache + _, resp = s.do("GET", "http://"+strings.Replace(pdh, "+", "-", 1)+".keep-web.example/missingfile", "", http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + "Cache-Control": {"must-revalidate"}, + }) + c.Check(resp.Code, check.Equals, http.StatusNotFound) +} - for _, forceReload := range []bool{false, true, false, true} { - _, err := cache.Get(arv, arvadostest.FooCollection, forceReload) - c.Check(err, check.Equals, nil) - } +func (s *IntegrationSuite) TestForceReloadUUID(c *check.C) { + client := arvados.NewClientFromEnv() + client.AuthToken = arvadostest.ActiveToken + var coll arvados.Collection + err := client.RequestAndDecode(&coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:oldfile\n", + }, + }) + c.Assert(err, check.IsNil) + defer client.RequestAndDecode(nil, "DELETE", "arvados/v1/collections/"+coll.UUID, nil, nil) - c.Check(cache.Stats().Requests, check.Equals, uint64(4)) - c.Check(cache.Stats().CollectionHits, check.Equals, uint64(1)) - c.Check(cache.Stats().PermissionHits, check.Equals, uint64(1)) - c.Check(cache.Stats().PDHHits, check.Equals, uint64(1)) - c.Check(cache.Stats().APICalls, check.Equals, uint64(3)) + _, resp := s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/oldfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusOK) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + err = client.RequestAndDecode(&coll, "PATCH", "arvados/v1/collections/"+coll.UUID, nil, map[string]interface{}{ + "collection": map[string]string{ + "manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:oldfile 0:0:newfile\n", + }, + }) + c.Assert(err, check.IsNil) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", arvadostest.ActiveToken, nil) + c.Check(resp.Code, check.Equals, http.StatusNotFound) + _, resp = s.do("GET", "http://"+coll.UUID+".keep-web.example/newfile", "", http.Header{ + "Authorization": {"Bearer " + arvadostest.ActiveToken}, + "Cache-Control": {"must-revalidate"}, + }) + c.Check(resp.Code, check.Equals, http.StatusOK) }