From 3faf9753f50715a635ebaba790564880e61fa035 Mon Sep 17 00:00:00 2001
From: Tom Clegg <tom@curii.com>
Date: Fri, 5 May 2023 10:03:00 -0400
Subject: [PATCH] Check /metrics & /_inspect/requests are available during busy
 times.

refs #20474

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@curii.com>
---
 lib/service/cmd_test.go | 35 +++++++++++++++++++++++++++++++++--
 1 file changed, 33 insertions(+), 2 deletions(-)

diff --git a/lib/service/cmd_test.go b/lib/service/cmd_test.go
index 22b8415137..5a7fbc0520 100644
--- a/lib/service/cmd_test.go
+++ b/lib/service/cmd_test.go
@@ -204,7 +204,7 @@ func (*Suite) TestDumpRequests(c *check.C) {
 	defer os.Remove(cf.Name())
 	defer cf.Close()
 
-	max := 1
+	max := 24
 	fmt.Fprintf(cf, `
 Clusters:
  zzzzz:
@@ -269,9 +269,9 @@ Clusters:
 		}
 	}
 	for {
+		time.Sleep(time.Second / 100)
 		j, err := os.ReadFile(tmpdir + "/arvados-controller-requests.json")
 		if os.IsNotExist(err) && deadline.After(time.Now()) {
-			time.Sleep(time.Second / 100)
 			continue
 		}
 		c.Check(err, check.IsNil)
@@ -281,10 +281,41 @@ Clusters:
 		var loaded []struct{ URL string }
 		err = json.Unmarshal(j, &loaded)
 		c.Check(err, check.IsNil)
+		if len(loaded) < max {
+			// Dumped when #requests was >90% but <100% of
+			// limit. If we stop now, we won't be able to
+			// confirm (below) that management endpoints
+			// are still accessible when normal requests
+			// are at 100%.
+			c.Logf("loaded dumped requests, but len %d < max %d -- still waiting", len(loaded), max)
+			continue
+		}
 		c.Check(loaded, check.HasLen, max)
 		c.Check(loaded[0].URL, check.Equals, "/testpath")
 		break
 	}
+
+	for _, path := range []string{"/_inspect/requests", "/metrics"} {
+		req, err := http.NewRequest("GET", "http://localhost:12345"+path, nil)
+		c.Assert(err, check.IsNil)
+		req.Header.Set("Authorization", "Bearer bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb")
+		resp, err := client.Do(req)
+		if !c.Check(err, check.IsNil) {
+			break
+		}
+		c.Logf("got response for %s", path)
+		c.Check(resp.StatusCode, check.Equals, http.StatusOK)
+		buf, err := ioutil.ReadAll(resp.Body)
+		c.Check(err, check.IsNil)
+		switch path {
+		case "/metrics":
+			c.Check(string(buf), check.Matches, `(?ms).*arvados_concurrent_requests `+fmt.Sprintf("%d", max)+`\n.*`)
+		case "/_inspect/requests":
+			c.Check(string(buf), check.Matches, `(?ms).*"URL":"/testpath".*`)
+		default:
+			c.Error("oops, testing bug")
+		}
+	}
 	close(hold)
 	cancel()
 
-- 
2.30.2