X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/9acc8cb9cd9ca4429712b0d31b647e9a6ecf2d96..52d652233f981839e4a8cfe25fadf985adda82ee:/services/keep-web/server_test.go

diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index dd8ce06172..0308f949f4 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -29,6 +29,7 @@ import (
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
 	"git.arvados.org/arvados.git/sdk/go/keepclient"
+	"github.com/prometheus/client_golang/prometheus"
 	check "gopkg.in/check.v1"
 )
 
@@ -47,19 +48,19 @@ func (s *IntegrationSuite) TestNoToken(c *check.C) {
 		"",
 		"bogustoken",
 	} {
-		hdr, body, _ := s.runCurl(c, token, "collections.example.com", "/collections/"+arvadostest.FooCollection+"/foo")
-		c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
-		c.Check(body, check.Equals, notFoundMessage+"\n")
+		hdr, body, _ := s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/c="+arvadostest.FooCollection+"/foo")
+		c.Check(hdr, check.Matches, `(?s)HTTP/1.1 401 Unauthorized\r\n.*`)
+		c.Check(strings.TrimSpace(body), check.Equals, unauthorizedMessage)
 
 		if token != "" {
-			hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
+			hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/collections/download/"+arvadostest.FooCollection+"/"+token+"/foo")
 			c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
-			c.Check(body, check.Equals, notFoundMessage+"\n")
+			c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
 		}
 
-		hdr, body, _ = s.runCurl(c, token, "collections.example.com", "/bad-route")
+		hdr, body, _ = s.runCurl(c, token, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, "/bad-route")
 		c.Check(hdr, check.Matches, `(?s)HTTP/1.1 404 Not Found\r\n.*`)
-		c.Check(body, check.Equals, notFoundMessage+"\n")
+		c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
 	}
 }
 
@@ -74,25 +75,21 @@ func (s *IntegrationSuite) Test404(c *check.C) {
 		"/download",
 		"/collections",
 		"/collections/",
-		// Implicit/generated index is not implemented yet;
-		// until then, return 404.
-		"/collections/" + arvadostest.FooCollection,
-		"/collections/" + arvadostest.FooCollection + "/",
-		"/collections/" + arvadostest.FooBarDirCollection + "/dir1",
-		"/collections/" + arvadostest.FooBarDirCollection + "/dir1/",
-		// Non-existent file in collection
-		"/collections/" + arvadostest.FooCollection + "/theperthcountyconspiracy",
+		// Non-existent file/directory
+		"/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy",
+		"/c=" + arvadostest.FooCollection + "/theperthcountyconspiracy/",
 		"/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
+		"/collections/download/" + arvadostest.FooCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy/",
 		// Non-existent collection
-		"/collections/" + arvadostest.NonexistentCollection,
-		"/collections/" + arvadostest.NonexistentCollection + "/",
-		"/collections/" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
+		"/c=" + arvadostest.NonexistentCollection,
+		"/c=" + arvadostest.NonexistentCollection + "/",
+		"/c=" + arvadostest.NonexistentCollection + "/theperthcountyconspiracy",
 		"/collections/download/" + arvadostest.NonexistentCollection + "/" + arvadostest.ActiveToken + "/theperthcountyconspiracy",
 	} {
-		hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, "collections.example.com", uri)
+		hdr, body, _ := s.runCurl(c, arvadostest.ActiveToken, s.handler.Cluster.Services.WebDAVDownload.ExternalURL.Host, uri)
 		c.Check(hdr, check.Matches, "(?s)HTTP/1.1 404 Not Found\r\n.*")
 		if len(body) > 0 {
-			c.Check(body, check.Equals, notFoundMessage+"\n")
+			c.Check(strings.TrimSpace(body), check.Equals, notFoundMessage)
 		}
 	}
 }
@@ -263,10 +260,14 @@ func (s *IntegrationSuite) Test200(c *check.C) {
 }
 
 // Return header block and body.
-func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
+func (s *IntegrationSuite) runCurl(c *check.C, auth, hostport, uri string, args ...string) (hdr, bodyPart string, bodySize int64) {
 	curlArgs := []string{"--silent", "--show-error", "--include"}
 	testHost, testPort, _ := net.SplitHostPort(s.testServer.URL[7:])
-	curlArgs = append(curlArgs, "--resolve", host+":"+testPort+":"+testHost)
+	host, port, _ := net.SplitHostPort(hostport)
+	if port == "" {
+		port = "80"
+	}
+	curlArgs = append(curlArgs, "--connect-to", host+":"+port+":"+testHost+":"+testPort)
 	if strings.Contains(auth, " ") {
 		// caller supplied entire Authorization header value
 		curlArgs = append(curlArgs, "-H", "Authorization: "+auth)
@@ -275,7 +276,7 @@ func (s *IntegrationSuite) runCurl(c *check.C, auth, host, uri string, args ...s
 		curlArgs = append(curlArgs, "-H", "Authorization: Bearer "+auth)
 	}
 	curlArgs = append(curlArgs, args...)
-	curlArgs = append(curlArgs, "http://"+host+":"+testPort+uri)
+	curlArgs = append(curlArgs, "http://"+hostport+uri)
 	c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 	cmd := exec.Command("curl", curlArgs...)
 	stdout, err := cmd.StdoutPipe()
@@ -411,6 +412,24 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 		resp.Body.Close()
 	}
 
+	var coll arvados.Collection
+	arv, err := arvadosclient.MakeArvadosClient()
+	c.Assert(err, check.IsNil)
+	arv.ApiToken = arvadostest.ActiveTokenV2
+	err = arv.Create("collections", map[string]interface{}{"ensure_unique_name": true}, &coll)
+	c.Assert(err, check.IsNil)
+	defer arv.Delete("collections", coll.UUID, nil, nil)
+	for i := 0; i < 2; i++ {
+		size := 1 << (i * 12)
+		req, _ = http.NewRequest("PUT", srvaddr+"/zero-"+fmt.Sprintf("%d", size), bytes.NewReader(make([]byte, size)))
+		req.Host = coll.UUID + ".example.com"
+		req.Header.Set("Authorization", "Bearer "+arvadostest.ActiveToken)
+		resp, err = http.DefaultClient.Do(req)
+		c.Assert(err, check.IsNil)
+		c.Check(resp.StatusCode, check.Equals, http.StatusCreated)
+		resp.Body.Close()
+	}
+
 	time.Sleep(metricsUpdateInterval * 2)
 
 	req, _ = http.NewRequest("GET", srvaddr+"/metrics.json", nil)
@@ -475,13 +494,7 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 	c.Check(summaries["request_duration_seconds/get/200"].SampleCount, check.Equals, "3")
 	c.Check(summaries["request_duration_seconds/get/404"].SampleCount, check.Equals, "1")
 	c.Check(summaries["time_to_status_seconds/get/404"].SampleCount, check.Equals, "1")
-	c.Check(counters["arvados_keepweb_collectioncache_requests//"].Value, check.Equals, int64(2))
-	c.Check(counters["arvados_keepweb_collectioncache_api_calls//"].Value, check.Equals, int64(2))
-	c.Check(counters["arvados_keepweb_collectioncache_hits//"].Value, check.Equals, int64(1))
-	c.Check(counters["arvados_keepweb_collectioncache_pdh_hits//"].Value, check.Equals, int64(1))
-	c.Check(gauges["arvados_keepweb_collectioncache_cached_manifests//"].Value, check.Equals, float64(1))
-	// FooCollection's cached manifest size is 45 ("1f4b0....+45") plus one 51-byte blob signature
-	c.Check(gauges["arvados_keepweb_sessions_cached_collection_bytes//"].Value, check.Equals, float64(45+51))
+	c.Check(gauges["arvados_keepweb_sessions_cached_session_bytes//"].Value, check.Equals, float64(1208))
 
 	// If the Host header indicates a collection, /metrics.json
 	// refers to a file in the collection -- the metrics handler
@@ -495,6 +508,22 @@ func (s *IntegrationSuite) TestMetrics(c *check.C) {
 		c.Assert(err, check.IsNil)
 		c.Check(resp.StatusCode, check.Equals, http.StatusNotFound)
 	}
+
+	req, _ = http.NewRequest("GET", srvaddr+"/metrics", nil)
+	req.Host = cluster.Services.WebDAVDownload.ExternalURL.Host
+	req.Header.Set("Authorization", "Bearer "+arvadostest.ManagementToken)
+	resp, err = http.DefaultClient.Do(req)
+	c.Assert(err, check.IsNil)
+	c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+	allmetrics, err := ioutil.ReadAll(resp.Body)
+	c.Check(err, check.IsNil)
+
+	c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_apparent_backend_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
+	c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_download_speed_bucket{size_range="0",le="\+Inf"} 4\n.*`)
+	c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_speed_bucket{size_range="0",le="\+Inf"} 2\n.*`)
+	c.Check(string(allmetrics), check.Matches, `(?ms).*\narvados_keepweb_upload_sync_delay_seconds_bucket{size_range="0",le="10"} 2\n.*`)
+
+	c.Logf("%s", allmetrics)
 }
 
 func (s *IntegrationSuite) SetUpSuite(c *check.C) {
@@ -527,7 +556,7 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
 
 	ctx := ctxlog.Context(context.Background(), logger)
 
-	s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, nil).(*handler)
+	s.handler = newHandlerOrErrorHandler(ctx, cluster, cluster.SystemRootToken, prometheus.NewRegistry()).(*handler)
 	s.testServer = httptest.NewUnstartedServer(
 		httpserver.AddRequestIDs(
 			httpserver.LogRequests(