'http://__CONTROLLER_INT_IP__:9006': {}
Keepbalance:
InternalURLs:
- 'http://localhost:9005': {}
+ 'http://__CONTROLLER_INT_IP__:9005': {}
Keepproxy:
ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__KEEP_EXT_SSL_PORT__'
InternalURLs:
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### LETSENCRYPT
+letsencrypt:
+ domainsets:
+ monitoring.__CLUSTER__.__DOMAIN__:
+ - mon.__CLUSTER__.__DOMAIN__
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+{%- import_yaml "ssl_key_encrypted.sls" as ssl_key_encrypted_pillar %}
+
+### NGINX
+nginx:
+ ### SERVER
+ server:
+ config:
+ ### STREAMS
+ http:
+ upstream prometheus_upstream:
+ - server: '127.0.0.1:9090 fail_timeout=10s'
+
+ ### SITES
+ servers:
+ managed:
+ ### PROMETHEUS
+ prometheus:
+ enabled: true
+ overwrite: true
+ config:
+ - server:
+ - server_name: mon.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 80
+ - location /.well-known:
+ - root: /var/www
+ - location /:
+ - return: '301 https://$host$request_uri'
+
+ prometheus-ssl:
+ enabled: true
+ overwrite: true
+ requires:
+ __CERT_REQUIRES__
+ config:
+ - server:
+ - server_name: mon.__CLUSTER__.__DOMAIN__
+ - listen:
+ - 443 http2 ssl
+ - index: index.html index.htm
+ - location /:
+ - proxy_pass: 'http://prometheus_upstream'
+ - proxy_read_timeout: 300
+ - proxy_connect_timeout: 90
+ - proxy_redirect: 'off'
+ - proxy_set_header: X-Forwarded-Proto https
+ - proxy_set_header: 'Host $http_host'
+ - proxy_set_header: 'X-Real-IP $remote_addr'
+ - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+ - ssl_certificate: __CERT_PEM__
+ - ssl_certificate_key: __CERT_KEY__
+ - include: snippets/ssl_hardening_default.conf
+ {%- if ssl_key_encrypted_pillar.ssl_key_encrypted.enabled %}
+ - ssl_password_file: {{ '/run/arvados/' | path_join(ssl_key_encrypted_pillar.ssl_key_encrypted.privkey_password_filename) }}
+ {%- endif %}
+ - access_log: /var/log/nginx/mon.__CLUSTER__.__DOMAIN__.access.log combined
+ - error_log: /var/log/nginx/mon.__CLUSTER__.__DOMAIN__.error.log
### POSTGRESQL
postgres:
+ pkgs_extra:
+ - postgresql-contrib
use_upstream_repo: true
version: '12'
postgresconf: |-
__CLUSTER___arvados:
ensure: present
password: "__DATABASE_PASSWORD__"
+ prometheus:
+ ensure: present
# tablespaces:
# arvados_tablespace:
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+### PROMETHEUS
+prometheus:
+ wanted:
+ component:
+ - postgres_exporter
--- /dev/null
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### PROMETHEUS
+prometheus:
+ wanted:
+ component:
+ - prometheus
+ - alertmanager
+ - blackbox_exporter
+ pkg:
+ use_upstream_repo: true
+ use_upstream_archive: true
+
+ component:
+ prometheus:
+ config:
+ global:
+ scrape_interval: 15s
+ evaluation_interval: 15s
+ rule_files:
+ - rules.yml
+
+ scrape_configs:
+ - job_name: prometheus
+ # metrics_path defaults to /metrics
+ # scheme defaults to http.
+ static_configs:
+ - targets: ['localhost:9090']
+ labels:
+ instance: mon.__CLUSTER__
+ cluster: __CLUSTER__
+
+ ## Arvados unique jobs
+ - job_name: keep_web
+ bearer_token: __MANAGEMENT_TOKEN__
+ scheme: https
+ static_configs:
+ - targets: ['keep.__CLUSTER__.__DOMAIN__:443']
+ labels:
+ instance: keep-web.__CLUSTER__
+ cluster: __CLUSTER__
+ - job_name: keep_balance
+ bearer_token: __MANAGEMENT_TOKEN__
+ static_configs:
+ - targets: ['__CONTROLLER_INT_IP__:9005']
+ labels:
+ instance: keep-balance.__CLUSTER__
+ cluster: __CLUSTER__
+ - job_name: keepstore
+ bearer_token: __MANAGEMENT_TOKEN__
+ static_configs:
+ - targets: ['__KEEPSTORE0_INT_IP__:25107']
+ labels:
+ instance: keep0.__CLUSTER__
+ cluster: __CLUSTER__
+ - targets: ['__KEEPSTORE1_INT_IP__:25107']
+ labels:
+ instance: keep1.__CLUSTER__
+ cluster: __CLUSTER__
+ - job_name: arvados_dispatch_cloud
+ bearer_token: __MANAGEMENT_TOKEN__
+ static_configs:
+ - targets: ['__CONTROLLER_INT_IP__:9006']
+ labels:
+ instance: arvados-dispatch-cloud.__CLUSTER__
+ cluster: __CLUSTER__
+
+ # Database
+ - job_name: postgresql
+ static_configs:
+ - targets: [
+ '__DATABASE_INT_IP__:9187',
+ '__DATABASE_INT_IP__:3903'
+ ]
+ labels:
+ instance: database.__CLUSTER__
+ cluster: __CLUSTER__
--- /dev/null
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+### PACKAGES
+monitoring_required_pkgs:
+ pkg.installed:
+ - name: mtail
+
+### FILES
+prometheus_pg_exporter_etc_default:
+ file.managed:
+ - name: /etc/default/prometheus-postgres-exporter
+ - contents: |
+ ### This file managed by Salt, do not edit by hand!!
+ #
+ # For details, check /usr/share/doc/prometheus-postgres-exporter/README.Debian
+ DATA_SOURCE_NAME='user=prometheus host=/run/postgresql dbname=postgres'
+ - require:
+ - pkg: prometheus-package-install-postgres_exporter-installed
+
+mtail_postgresql_conf:
+ file.managed:
+ - name: /etc/mtail/postgresql.mtail
+ - contents: |
+ ########################################################################
+ # File managed by Salt.
+ # Your changes will be overwritten.
+ ########################################################################
+
+ # Parser for postgresql's log statement duration
+
+ gauge postgresql_statement_duration_seconds by statement
+
+ /^/ +
+ /(?P<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} (\w+)) / + # 2019-01-16 16:53:45 GMT
+ /LOG: +duration: / +
+ /(?P<duration>[0-9\.]+) ms/ + # 153.967 ms
+ /(.*?): (?P<statement>.+)/ + # statement: SELECT COUNT(*) FROM (SELECT rolname FROM pg_roles WHERE rolname='arvados') count
+ /$/ {
+ strptime($timestamp, "2006-01-02 15:04:05 MST") # for tests
+
+ postgresql_statement_duration_seconds[$statement] = $duration / 1000
+ }
+ - require:
+ - pkg: monitoring_required_pkgs
+
+mtail_etc_default:
+ file.managed:
+ - name: /etc/default/mtail
+ - contents: |
+ ### This file managed by Salt, do not edit by hand!!
+ #
+ ENABLED=true
+ # List of files to monitor (mandatory).
+ LOGS=/var/log/postgresql/postgresql*log
+ - require:
+ - pkg: monitoring_required_pkgs
+
+### SERVICES
+prometheus_pg_exporter_service:
+ service.running:
+ - name: prometheus-postgres-exporter
+ - enable: true
+ - require:
+ - pkg: prometheus-package-install-postgres_exporter-installed
+ - watch:
+ - file: /etc/default/prometheus-postgres-exporter
+
+mtail_service:
+ service.running:
+ - name: mtail
+ - enable: true
+ - require:
+ - pkg: monitoring_required_pkgs
+ - watch:
+ - file: /etc/mtail/postgresql.mtail
+ - file: /etc/default/mtail
else
# Just deploy the node that was supplied on the command line.
sync $NODE $BRANCH
- deploynode $NODE ""
+ deploynode $NODE "${NODES[$NODE]}"
fi
set +x
# DOCKER_TAG="v2.4.2"
# LOCALE_TAG="v0.3.4"
# LETSENCRYPT_TAG="v2.1.0"
+# PROMETHEUS_TAG="v5.6.5"
\ No newline at end of file
echo >&2 " keepbalance"
echo >&2 " keepstore"
echo >&2 " keepweb"
+ echo >&2 " monitoring"
echo >&2 " shell"
echo >&2 " webshell"
echo >&2 " websocket"
for i in ${2//,/ }
do
# Verify the role exists
- if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
+ if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher,monitoring" == *"$i"* ]]; then
echo "The role '${i}' is not a valid role"
usage
exit 1
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
LOGROTATE_TAG="v0.14.0"
+PROMETHEUS_TAG="v5.6.5"
# Salt's dir
DUMP_SALT_CONFIG_DIR=""
|| git clone --quiet ${POSTGRES_URL} ${F_DIR}/postgres
( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" )
+echo "...prometheus"
+test -d prometheus && ( cd prometheus && git fetch ) \
+ || git clone --quiet https://github.com/saltstack-formulas/prometheus-formula.git ${F_DIR}/prometheus
+( cd prometheus && git checkout --quiet tags/"${PROMETHEUS_TAG}" )
+
echo "...letsencrypt"
test -d letsencrypt && ( cd letsencrypt && git fetch ) \
|| git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
case "${R}" in
"database")
# States
- echo " - postgres" >> ${S_DIR}/top.sls
+ grep -q "\- postgres$" ${S_DIR}/top.sls || echo " - postgres" >> ${S_DIR}/top.sls
+ grep -q "prometheus" ${S_DIR}/top.sls || echo " - prometheus" >> ${S_DIR}/top.sls
+ grep -q "extra.postgresql_mtail" ${S_DIR}/top.sls || echo " - extra.postgresql_mtail" >> ${S_DIR}/top.sls
# Pillars
- echo ' - postgresql' >> ${P_DIR}/top.sls
+ grep -q "postgresql" ${P_DIR}/top.sls || echo " - postgresql" >> ${P_DIR}/top.sls
+ grep -q "prometheus_pg_exporter" ${P_DIR}/top.sls || echo " - prometheus_pg_exporter" >> ${P_DIR}/top.sls
+ ;;
+ "monitoring")
+ ### States ###
+ grep -q "nginx" ${S_DIR}/top.sls || echo " - nginx" >> ${S_DIR}/top.sls
+ grep -q "prometheus" ${S_DIR}/top.sls || echo " - prometheus" >> ${S_DIR}/top.sls
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ elif [ "${SSL_MODE}" = "bring-your-own" ]; then
+ copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
+ if [ "${SSL_KEY_ENCRYPTED}" = "yes" ]; then
+ grep -q "ssl_key_encrypted" ${S_DIR}/top.sls || echo " - extra.ssl_key_encrypted" >> ${S_DIR}/top.sls
+ fi
+ fi
+ ### Pillars ###
+ grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo " - nginx_${R}_configuration" >> ${P_DIR}/top.sls
+ grep -q "prometheus_server" ${P_DIR}/top.sls || echo " - prometheus_server" >> ${P_DIR}/top.sls
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ grep -q "letsencrypt_${R}_configuration" ${P_DIR}/top.sls || echo " - letsencrypt_${R}_configuration" >> ${P_DIR}/top.sls
+ if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
+ fi
+ sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${R}.${CLUSTER}.${DOMAIN}*/g;
+ s#__CERT_PEM__#/etc/letsencrypt/live/${R}.${CLUSTER}.${DOMAIN}/fullchain.pem#g;
+ s#__CERT_KEY__#/etc/letsencrypt/live/${R}.${CLUSTER}.${DOMAIN}/privkey.pem#g" \
+ ${P_DIR}/nginx_${R}_configuration.sls
+ elif [ "${SSL_MODE}" = "bring-your-own" ]; then
+ grep -q "ssl_key_encrypted" ${P_DIR}/top.sls || echo " - ssl_key_encrypted" >> ${P_DIR}/top.sls
+ sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g;
+ s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g;
+ s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \
+ ${P_DIR}/nginx_${R}_configuration.sls
+ grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls
+ fi
;;
"api")
# States
# SPDX-License-Identifier: CC-BY-SA-3.0
region_name = "us-east-1"
-# cluster_name = "xarv1"
-# domain_name = "example.com"
+cluster_name = "xarv1"
+domain_name = "example.com"
projects / arvados.git / commitdiff