20602: Report queue time for pos/neg/zero-priority reqs.

[arvados.git] / lib / service / cmd.go

diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 119d66191f889743de2f2407b423ca5f22663c71..f4d4ec8efe4b8878d97fafdfc70ba6e364f92689 100644 (file)
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -157,6 +157,8 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
                                                        &httpserver.RequestLimiter{
                                                                Handler:       handler,
                                                                MaxConcurrent: cluster.API.MaxConcurrentRequests,
+                                                               MaxQueue:      cluster.API.MaxQueuedRequests,
+                                                               Priority:      c.requestPriority,
                                                                Registry:      reg}))))))
        srv := &httpserver.Server{
                Server: http.Server{
@@ -210,7 +212,7 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
 // JSON file in the specified directory.
 func (c *command) requestQueueDumpCheck(cluster *arvados.Cluster, prog string, reg *prometheus.Registry, srv *http.Server, logger logrus.FieldLogger) {
        outdir := cluster.SystemLogs.RequestQueueDumpDirectory
-       if outdir == "" || cluster.ManagementToken == "" {
+       if outdir == "" || cluster.ManagementToken == "" || cluster.API.MaxConcurrentRequests < 1 {
                return
        }
        logger = logger.WithField("worker", "RequestQueueDump")
@@ -253,6 +255,29 @@ func (c *command) requestQueueDumpCheck(cluster *arvados.Cluster, prog string, r
        }
 }
 
+func (c *command) requestPriority(req *http.Request, queued time.Time) int64 {
+       switch {
+       case req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/containers/") && strings.HasSuffix(req.URL.Path, "/lock"):
+               // Return 503 immediately instead of queueing. We want
+               // to send feedback to dispatchcloud ASAP to stop
+               // bringing up new containers.
+               return httpserver.IneligibleForQueuePriority
+       case req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/logs"):
+               // "Create log entry" is the most harmless kind of
+               // request to drop. Negative priority is called "low"
+               // in aggregate metrics.
+               return -1
+       case req.Header.Get("Origin") != "":
+               // Handle interactive requests first. Positive
+               // priority is called "high" in aggregate metrics.
+               return 1
+       default:
+               // Zero priority is called "normal" in aggregate
+               // metrics.
+               return 0
+       }
+}
+
 // If an incoming request's target vhost has an embedded collection
 // UUID or PDH, handle it with hTrue, otherwise handle it with
 // hFalse.