20200: Add limiter for log create requests

Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz@curii.com>

diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 527439571ea97145e100457527bdf2e43643fc45..32a1b03135df8c0067ed1ec682a9267f18a051af 100644 (file)
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -225,7 +225,12 @@ Clusters:
 
       # Maximum number of concurrent requests to accept in a single
       # service process, or 0 for no limit.
-      MaxConcurrentRequests: 0
+      MaxConcurrentRequests: 64
+
+      # Fraction of MaxConcurrentRequests that can be "log create"
+      # messages at any given time.  This is to prevent logging
+      # updates from crowding out more important requests.
+      LogCreateRequestFraction: 0.50
 
       # Maximum number of 64MiB memory buffers per Keepstore server process, or
       # 0 for no limit. When this limit is reached, up to

diff --git a/lib/config/export.go b/lib/config/export.go
index a8a535eeb58540799be38e7a8fc852ad0c0552fe..44fd559418ea177f365921c00b89c76a1a4088b8 100644 (file)
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -68,6 +68,7 @@ var whitelist = map[string]bool{
        "API.LockBeforeUpdate":                     false,
        "API.KeepServiceRequestTimeout":            false,
        "API.MaxConcurrentRequests":                false,
+       "API.LogCreateRequestFraction":             false,
        "API.MaxIndexDatabaseRead":                 false,
        "API.MaxItemsPerResponse":                  true,
        "API.MaxKeepBlobBuffers":                   false,

diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 4c6fca7f77276c3981c591d18429d8520d3e76b7..07e6514810f97ceb691be7f9c528fd5560694d66 100644 (file)
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -38,6 +38,7 @@ type Handler struct {
        secureClient   *http.Client
        insecureClient *http.Client
        dbConnector    ctrlctx.DBConnector
+       limitLogCreate chan struct{}
 }
 
 func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
@@ -147,6 +148,17 @@ func (h *Handler) setup() {
        ic.CheckRedirect = neverRedirect
        h.insecureClient = &ic
 
+       logCreateLimit := int(float64(h.Cluster.API.MaxConcurrentRequests) * h.Cluster.API.LogCreateRequestFraction)
+       if logCreateLimit == 0 && h.Cluster.API.LogCreateRequestFraction > 0 {
+               logCreateLimit = 1
+       }
+       if logCreateLimit == 0 {
+               // can't have unlimited size channels, so just make
+               // the buffer size really big.
+               logCreateLimit = 4096
+       }
+       h.limitLogCreate = make(chan struct{}, logCreateLimit)
+
        h.proxy = &proxy{
                Name: "arvados-controller",
        }
@@ -182,6 +194,20 @@ func (h *Handler) localClusterRequest(req *http.Request) (*http.Response, error)
        return h.proxy.Do(req, urlOut, client)
 }
 
+func (h *Handler) limitLogCreateRequests(w http.ResponseWriter, req *http.Request, next http.Handler) {
+       if req.Method == http.MethodPost && strings.HasPrefix(req.URL.Path, "/arvados/v1/logs") {
+               select {
+               case h.limitLogCreate <- struct{}{}:
+                       next.ServeHTTP(w, req)
+                       <-h.limitLogCreate
+               default:
+                       http.Error(w, "Excess log messages", http.StatusServiceUnavailable)
+               }
+               return
+       }
+       next.ServeHTTP(w, req)
+}
+
 func (h *Handler) proxyRailsAPI(w http.ResponseWriter, req *http.Request, next http.Handler) {
        resp, err := h.localClusterRequest(req)
        n, err := h.proxy.ForwardResponse(w, resp, err)

diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index bf70a76032287fb08b862f18a00b9aacab450022..4466b0a4deffda52a71d9084dc324c1a1df807e6 100644 (file)
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -100,6 +100,7 @@ type Cluster struct {
                MaxIndexDatabaseRead             int
                MaxItemsPerResponse              int
                MaxConcurrentRequests            int
+               LogCreateRequestFraction         float64
                MaxKeepBlobBuffers               int
                MaxRequestAmplification          int
                MaxRequestSize                   int