From: Lucas Di Pentima Date: Fri, 12 Jan 2024 21:27:39 +0000 (-0300) Subject: Merge branch '21285-installer-updates'. Closes #21285 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/01ca27ba0a1ef84c53e223004249505435a788b6?hp=f9ae14ce8c4d749cfdc007ec45929cc9a09a3790 Merge branch '21285-installer-updates'. Closes #21285 Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- diff --git a/tools/salt-install/config_examples/multi_host/aws/dashboards/arvados_overview.json b/tools/salt-install/config_examples/multi_host/aws/dashboards/arvados_overview.json index 774109b531..3b3fd83270 100644 --- a/tools/salt-install/config_examples/multi_host/aws/dashboards/arvados_overview.json +++ b/tools/salt-install/config_examples/multi_host/aws/dashboards/arvados_overview.json @@ -1052,7 +1052,7 @@ }, "expr": "arvados_concurrent_requests{}", "interval": "", - "legendFormat": "{{instance}}", + "legendFormat": "{{instance}}_{{queue}}", "refId": "A" } ], diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls index bd95c5a868..16e686ab80 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls @@ -6,6 +6,7 @@ {%- set _workers = ("__CONTROLLER_MAX_WORKERS__" or grains['num_cpus']*2)|int %} {%- set max_workers = [_workers, 8]|max %} {%- set max_reqs = ("__CONTROLLER_MAX_QUEUED_REQUESTS__" or 128)|int %} +{%- set max_tunnels = ("__CONTROLLER_MAX_GATEWAY_TUNNELS__" or 1000)|int %} {%- set database_host = ("__DATABASE_EXTERNAL_SERVICE_HOST_OR_IP__" or "__DATABASE_INT_IP__") %} {%- set database_name = "__DATABASE_NAME__" %} {%- set database_user = "__DATABASE_USER__" %} @@ -121,6 +122,7 @@ arvados: MaxConcurrentRailsRequests: {{ max_workers * 2 }} MaxConcurrentRequests: {{ max_reqs }} MaxQueuedRequests: {{ max_reqs }} + MaxGatewayTunnels: {{ max_tunnels }} ### CONTAINERS {%- set dispatcher_ssh_privkey = "__DISPATCHER_SSH_PRIVKEY__" %} diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_balancer_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_balancer_configuration.sls index a3a3d405f1..485cf9c12f 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_balancer_configuration.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_balancer_configuration.sls @@ -9,21 +9,16 @@ {%- set controller_nr = balancer_backends|length %} {%- set disabled_controller = "__DISABLED_CONTROLLER__" %} {%- set max_reqs = ("__CONTROLLER_MAX_QUEUED_REQUESTS__" or 128)|int %} +{%- set max_tunnels = ("__CONTROLLER_MAX_GATEWAY_TUNNELS__" or 1000)|int %} ### NGINX nginx: ### SERVER server: config: - {%- if max_reqs != "" %} - worker_rlimit_nofile: {{ (max_reqs|int * 3 * controller_nr)|round|int }} + worker_rlimit_nofile: {{ (max_reqs + max_tunnels) * 5 * controller_nr }} events: - worker_connections: {{ (max_reqs|int * 3 * controller_nr)|round|int }} - {%- else %} - worker_rlimit_nofile: 4096 - events: - worker_connections: 1024 - {%- endif %} + worker_connections: {{ (max_reqs + max_tunnels) * 5 * controller_nr }} ### STREAMS http: 'geo $external_client': diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls index 82f1b91bb5..0c9ef1c36e 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls @@ -15,6 +15,7 @@ {%- set _workers = ("__CONTROLLER_MAX_WORKERS__" or grains['num_cpus']*2)|int %} {%- set max_workers = [_workers, 8]|max %} {%- set max_reqs = ("__CONTROLLER_MAX_QUEUED_REQUESTS__" or 128)|int %} +{%- set max_tunnels = ("__CONTROLLER_MAX_GATEWAY_TUNNELS__" or 1000)|int %} ### NGINX nginx: @@ -50,13 +51,14 @@ nginx: {% endif %} worker_processes: {{ max_workers }} - # each request is up to 3 connections (1 with client, 1 proxy to + # Each client request is up to 3 connections (1 with client, 1 proxy to # controller, then potentially 1 from controller back to # passenger). Each connection consumes a file descriptor. # That's how we get these calculations - worker_rlimit_nofile: {{ max_reqs * 3 + 1 }} + # (we're multiplying by 5 instead to be on the safe side) + worker_rlimit_nofile: {{ (max_reqs + max_tunnels) * 5 + 1 }} events: - worker_connections: {{ max_reqs * 3 + 1 }} + worker_connections: {{ (max_reqs + max_tunnels) * 5 + 1 }} ### SITES servers: diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts index 909c3354ec..d97afaca1c 100644 --- a/tools/salt-install/local.params.example.multiple_hosts +++ b/tools/salt-install/local.params.example.multiple_hosts @@ -148,9 +148,11 @@ DATABASE_USER="${CLUSTER}_arvados" #DATABASE_POSTGRESQL_VERSION= # Performance tuning parameters. If these are not set, workers -# defaults on the number of cpus and queued requests defaults to 128. +# defaults on the number of cpus, queued requests defaults to 128 +# and gateway tunnels defaults to 1000. #CONTROLLER_MAX_WORKERS= #CONTROLLER_MAX_QUEUED_REQUESTS= +#CONTROLLER_MAX_GATEWAY_TUNNELS= # The directory to check for the config files (pillars, states) you want to use. # There are a few examples under 'config_examples'. diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh index bb4a4c6eb5..eb09dddf2c 100755 --- a/tools/salt-install/provision.sh +++ b/tools/salt-install/provision.sh @@ -206,6 +206,7 @@ apply_var_substitutions() { s#__SSL_KEY_AWS_SECRET_NAME__#${SSL_KEY_AWS_SECRET_NAME}#g; s#__CONTROLLER_MAX_WORKERS__#${CONTROLLER_MAX_WORKERS:-}#g; s#__CONTROLLER_MAX_QUEUED_REQUESTS__#${CONTROLLER_MAX_QUEUED_REQUESTS:-128}#g; + s#__CONTROLLER_MAX_GATEWAY_TUNNELS__#${CONTROLLER_MAX_GATEWAY_TUNNELS:-1000}#g; s#__MONITORING_USERNAME__#${MONITORING_USERNAME}#g; s#__MONITORING_EMAIL__#${MONITORING_EMAIL}#g; s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g;