X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/d6e1bfee59569d79f0f3e24620280e70aa161403..6bf9e1a4b5640f3cdd057810f0c9b8a945bb88bd:/services/keep-web/server_test.go diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go index 6688cc2ee7..acdc11b305 100644 --- a/services/keep-web/server_test.go +++ b/services/keep-web/server_test.go @@ -5,6 +5,7 @@ package main import ( + "bytes" "crypto/md5" "encoding/json" "fmt" @@ -17,10 +18,12 @@ import ( "strings" "testing" - "git.curoverse.com/arvados.git/sdk/go/arvados" - "git.curoverse.com/arvados.git/sdk/go/arvadosclient" - "git.curoverse.com/arvados.git/sdk/go/arvadostest" - "git.curoverse.com/arvados.git/sdk/go/keepclient" + "git.arvados.org/arvados.git/lib/config" + "git.arvados.org/arvados.git/sdk/go/arvados" + "git.arvados.org/arvados.git/sdk/go/arvadosclient" + "git.arvados.org/arvados.git/sdk/go/arvadostest" + "git.arvados.org/arvados.git/sdk/go/ctxlog" + "git.arvados.org/arvados.git/sdk/go/keepclient" check "gopkg.in/check.v1" ) @@ -148,7 +151,7 @@ type curlCase struct { } func (s *IntegrationSuite) Test200(c *check.C) { - s.testServer.Config.AnonymousTokens = []string{arvadostest.AnonymousToken} + s.testServer.Config.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken for _, spec := range []curlCase{ // My collection { @@ -164,16 +167,16 @@ func (s *IntegrationSuite) Test200(c *check.C) { dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { - host: strings.Replace(arvadostest.FooPdh, "+", "-", 1) + ".collections.example.com", + host: strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + ".collections.example.com", path: "/t=" + arvadostest.ActiveToken + "/foo", dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { - path: "/c=" + arvadostest.FooPdh + "/t=" + arvadostest.ActiveToken + "/foo", + path: "/c=" + arvadostest.FooCollectionPDH + "/t=" + arvadostest.ActiveToken + "/foo", dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { - path: "/c=" + strings.Replace(arvadostest.FooPdh, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo", + path: "/c=" + strings.Replace(arvadostest.FooCollectionPDH, "+", "-", 1) + "/t=" + arvadostest.ActiveToken + "/_/foo", dataMD5: "acbd18db4cc2f85cedef654fccc4a4d8", }, { @@ -266,9 +269,10 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs)) cmd := exec.Command("curl", curlArgs...) stdout, err := cmd.StdoutPipe() - c.Assert(err, check.Equals, nil) - cmd.Stderr = cmd.Stdout - go cmd.Start() + c.Assert(err, check.IsNil) + cmd.Stderr = os.Stderr + err = cmd.Start() + c.Assert(err, check.IsNil) buf := make([]byte, 2<<27) n, err := io.ReadFull(stdout, buf) // Discard (but measure size of) anything past 128 MiB. @@ -276,9 +280,9 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... if err == io.ErrUnexpectedEOF { buf = buf[:n] } else { - c.Assert(err, check.Equals, nil) + c.Assert(err, check.IsNil) discarded, err = io.Copy(ioutil.Discard, stdout) - c.Assert(err, check.Equals, nil) + c.Assert(err, check.IsNil) } err = cmd.Wait() // Without "-f", curl exits 0 as long as it gets a valid HTTP @@ -297,6 +301,7 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ... } func (s *IntegrationSuite) TestMetrics(c *check.C) { + s.testServer.Config.cluster.Services.WebDAVDownload.ExternalURL.Host = s.testServer.Addr origin := "http://" + s.testServer.Addr req, _ := http.NewRequest("GET", origin+"/notfound", nil) _, err := http.DefaultClient.Do(req) @@ -306,26 +311,45 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) { resp, err := http.DefaultClient.Do(req) c.Assert(err, check.IsNil) c.Check(resp.StatusCode, check.Equals, http.StatusOK) - req, _ = http.NewRequest("GET", origin+"/foo", nil) - req.Host = arvadostest.FooCollection + ".example.com" - req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken) + for i := 0; i < 2; i++ { + req, _ = http.NewRequest("GET", origin+"/foo", nil) + req.Host = arvadostest.FooCollection + ".example.com" + req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken) + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusOK) + buf, _ := ioutil.ReadAll(resp.Body) + c.Check(buf, check.DeepEquals, []byte("foo")) + resp.Body.Close() + } + + s.testServer.Config.Cache.updateGauges() + + req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) resp, err = http.DefaultClient.Do(req) c.Assert(err, check.IsNil) - c.Check(resp.StatusCode, check.Equals, http.StatusOK) - buf, _ := ioutil.ReadAll(resp.Body) - c.Check(buf, check.DeepEquals, []byte("foo")) + c.Check(resp.StatusCode, check.Equals, http.StatusUnauthorized) req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) + req.Header.Set("Authorization", "Bearer badtoken") + resp, err = http.DefaultClient.Do(req) + c.Assert(err, check.IsNil) + c.Check(resp.StatusCode, check.Equals, http.StatusForbidden) + + req, _ = http.NewRequest("GET", origin+"/metrics.json", nil) + req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken) resp, err = http.DefaultClient.Do(req) c.Assert(err, check.IsNil) c.Check(resp.StatusCode, check.Equals, http.StatusOK) type summary struct { - SampleCount string `json:"sample_count"` - SampleSum float64 `json:"sample_sum"` - Quantile []struct { - Quantile float64 - Value float64 - } + SampleCount string + SampleSum float64 + } + type counter struct { + Value int64 + } + type gauge struct { + Value float64 } var ents []struct { Name string @@ -336,24 +360,38 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) { Name string Value string } + Counter counter + Gauge gauge Summary summary } } json.NewDecoder(resp.Body).Decode(&ents) - flat := map[string]summary{} + summaries := map[string]summary{} + gauges := map[string]gauge{} + counters := map[string]counter{} for _, e := range ents { for _, m := range e.Metric { labels := map[string]string{} for _, lbl := range m.Label { labels[lbl.Name] = lbl.Value } - flat[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary + summaries[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Summary + counters[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Counter + gauges[e.Name+"/"+labels["method"]+"/"+labels["code"]] = m.Gauge } } - c.Check(flat["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0) - c.Check(flat["request_duration_seconds/get/200"].SampleCount, check.Equals, "2") - c.Check(flat["request_duration_seconds/get/404"].SampleCount, check.Equals, "1") - c.Check(flat["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1") + c.Check(summaries["request_duration_seconds/get/200"].SampleSum, check.Not(check.Equals), 0) + c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3") + c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1") + c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1") + c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2)) + c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(1)) + c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1)) + c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1)) + c.Check(counters["arvados_keepweb_collectioncache_permission_hits//"].Value, check.Equals, int64(1)) + c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1)) + // FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature + c.Check(gauges["arvados_keepweb_collectioncache_cached_manifest_bytes//"].Value, check.Equals, float64(45+51)) // If the Host header indicates a collection, /metrics.json // refers to a file in the collection -- the metrics handler @@ -378,6 +416,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) { kc.PutB([]byte("Hello world\n")) kc.PutB([]byte("foo")) kc.PutB([]byte("foobar")) + kc.PutB([]byte("waz")) } func (s *IntegrationSuite) TearDownSuite(c *check.C) { @@ -387,14 +426,24 @@ func (s *IntegrationSuite) TearDownSuite(c *check.C) { func (s *IntegrationSuite) SetUpTest(c *check.C) { arvadostest.ResetEnv() - cfg := DefaultConfig() + ldr := config.NewLoader(bytes.NewBufferString("Clusters: {zzzzz: {}}"), ctxlog.TestLogger(c)) + ldr.Path = "-" + arvCfg, err := ldr.Load() + c.Check(err, check.IsNil) + cfg := newConfig(arvCfg) + c.Assert(err, check.IsNil) cfg.Client = arvados.Client{ APIHost: testAPIHost, Insecure: true, } - cfg.Listen = "127.0.0.1:0" + listen := "127.0.0.1:0" + cfg.cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{} + cfg.cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: listen}] = arvados.ServiceInstance{} + cfg.cluster.ManagementToken = arvadostest.ManagementToken + cfg.cluster.SystemRootToken = arvadostest.SystemRootToken + cfg.cluster.Users.AnonymousUserToken = arvadostest.AnonymousToken s.testServer = &server{Config: cfg} - err := s.testServer.Start() + err = s.testServer.Start(ctxlog.TestLogger(c)) c.Assert(err, check.Equals, nil) }