X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/f98e0188777b3e2d229c968824b3e64307dae4e6..6bf9e1a4b5640f3cdd057810f0c9b8a945bb88bd:/services/keep-web/server_test.go diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go index 02f03d04af..acdc11b305 100644 --- a/services/keep-web/server_test.go +++ b/services/keep-web/server_test.go @@ -5,20 +5,25 @@ package main import ( + "bytes" "crypto/md5" + "encoding/json" "fmt" "io" "io/ioutil" "net" + "net/http" "os" "os/exec" "strings" "testing" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) @@ -59,7 +64,6 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) { func (s *IntegrationSuite) Test404(c *check.C) { for _, uri := range []string{ // Routing errors (always 404 regardless of what's stored in Keep) - "/", "/foo", "/download", "/collections", @@ -147,7 +151,7 @@ type curlCase struct { } func (s *IntegrationSuite) Test200(c *check.C) { - s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken} + s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken for _, spec := range []curlCase{ // My collection { @@ -163,16 +167,16 @@ func (s *IntegrationSuite) Test200(c *check.C) { dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { - host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com", + host: strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com", path: "/t=" + arvadostest.ActiveToken + "/foo", dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { - path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo", + path: "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo", dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { - path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo", + path: "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo", dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { @@ -265,9 +269,10 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs)) cmd := exec.Command("curl", curlArgs...) stdout, err := cmd.StdoutPipe() - c.Assert(err, check.Equals, nil) - cmd.Stderr = cmd.Stdout - go cmd.Start() + c.Assert(err, check.IsNil) + cmd.Stderr = os.Stderr + err = cmd.Start() + c.Assert(err, check.IsNil) buf := make([]byte, 2<<27) n, err := io.ReadFull(stdout, buf) // Discard (but measure size of) anything past 128 MiB. @@ -275,9 +280,9 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... if err == io.ErrUnexpectedEOF { buf = buf[:n] } else { - c.Assert(err, check.Equals, nil) + c.Assert(err, check.IsNil) discarded, err = io.Copy(ioutil.Discard, stdout) - c.Assert(err, check.Equals, nil) + c.Assert(err, check.IsNil) } err = cmd.Wait() // Without "-f", curl exits 0 as long as it gets a valid HTTP @@ -295,6 +300,110 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... return } +func (s *IntegrationSuite) TestMetrics(c *check.C) { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr + origin := "http://" + s.testServer.Addr + req, _ := http.NewRequest("GET", origin+"/notfound", nil) + _, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + req, _ = http.NewRequest("GET", origin+"/by_id/", nil) + req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken) + resp, err := http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + for i := 0; i < 2; i++ { + req, _ = http.NewRequest("GET", origin+"/foo", nil) + req.Host = arvadostest.FooCollection + ".example.com" + req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken) + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + buf, _ := ioutil.ReadAll(resp.Body) + c.Check(buf, check.DeepEquals, []byte("foo")) + resp.Body.Close() + } + + s.testServer.Config.Cache.updateGauges() + + req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized) + + req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) + req.Header.Set("Authorization", "Bearer badtoken") + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusForbidden) + + req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) + req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken) + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + type summary struct { + SampleCount string + SampleSum float64 + } + type counter struct { + Value int64 + } + type gauge struct { + Value float64 + } + var ents []struct { + Name string + Help string + Type string + Metric []struct { + Label []struct { + Name string + Value string + } + Counter counter + Gauge gauge + Summary summary + } + } + json.NewDecoder(resp.Body).Decode(&ents) + summaries := map[string]summary{} + gauges := map[string]gauge{} + counters := map[string]counter{} + for _, e := range ents { + for _, m := range e.Metric { + labels := map[string]string{} + for _, lbl := range m.Label { + labels[lbl.Name] = lbl.Value + } + summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary + counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter + gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge + } + } + c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0) + c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3") + c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1") + c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1") + c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2)) + c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1)) + c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1)) + c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1)) + c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1)) + c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1)) + // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature + c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51)) + + // If the Host header indicates a collection, /metrics.json + // refers to a file in the collection -- the metrics handler + // must not intercept that route. + req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) + req.Host = strings.Replace(arvadostest.FooCollectionPDH, "+", "-", -1) + ".example.com" + req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken) + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusNotFound) +} + func (s *IntegrationSuite) SetUpSuite(c *check.C) { arvadostest.StartAPI() arvadostest.StartKeep(2, true) @@ -307,6 +416,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { kc.PutB([]byte("Hello world\n")) kc.PutB([]byte("foo")) kc.PutB([]byte("foobar")) + kc.PutB([]byte("waz")) } func (s *IntegrationSuite) TearDownSuite(c *check.C) { @@ -316,14 +426,24 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) { func (s *IntegrationSuite) SetUpTest(c *check.C) { arvadostest.ResetEnv() - cfg := DefaultConfig() + ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c)) + ldr.Path = "-" + arvCfg, err := ldr.Load() + c.Check(err, check.IsNil) + cfg := newConfig(arvCfg) + c.Assert(err, check.IsNil) cfg.Client = arvados.Client{ APIHost: testAPIHost, Insecure: true, } - cfg.Listen = "127.0.0.1:0" + listen := "127.0.0.1:0" + cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{} + cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{} + cfg.cluster.ManagementToken = arvadostest.ManagementToken + cfg.cluster.SystemRootToken = arvadostest.SystemRootToken + cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testServer = &server{Config: cfg} - err := s.testServer.Start() + err = s.testServer.Start(ctxlog.TestLogger(c)) c.Assert(err, check.Equals, nil) }