16379: Adds prometheus' node_exporter to all nodes.

Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima@curii.com>

diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
index 2eed52a1db93b80fad73051d00d22195d37d8dcb..10cbb6c34ea73b40fd1d61269fd13317b2425d0c 100644 (file)
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
@@ -24,19 +24,12 @@ postgres:
       password: "__DATABASE_PASSWORD__"
     prometheus:
       ensure: present
-
-  # tablespaces:
-  #   arvados_tablespace:
-  #     directory: /path/to/some/tbspace/arvados_tbsp
-  #     owner: arvados
-
   databases:
     __CLUSTER___arvados:
       owner: __CLUSTER___arvados
       template: template0
       lc_ctype: en_US.utf8
       lc_collate: en_US.utf8
-      # tablespace: arvados_tablespace
       schemas:
         public:
           owner: __CLUSTER___arvados

diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_node_exporter.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_node_exporter.sls
new file mode 100644 (file)
index 0000000..74a5664
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_node_exporter.sls
@@ -0,0 +1,17 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+### PROMETHEUS
+prometheus:
+  wanted:
+    component:
+      - node_exporter
+  pkg:
+    use_upstream_repo: true
+    component:
+      node_exporter:
+        service:
+          args:
+            collector.textfile.directory: /var/lib/prometheus/node-exporter

diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_pg_exporter.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_pg_exporter.sls
index 73f706d16a3bab2594869455fcabe3e1043a74a3..62f654e525e8a0ee5b8510fb4eb32bf359a27f6c 100644 (file)
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_pg_exporter.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_pg_exporter.sls
@@ -1,9 +1,14 @@
+---
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
 
+prometheus_pg_exporter:
+  enabled: true
+
 ### PROMETHEUS
 prometheus:
   wanted:
     component:
       - postgres_exporter
+      - node_exporter

diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_server.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_server.sls
index dd5594dd883aa6c2137b2c17ea4bcf2d433bdd89..3988f3c126f3e81e6c6906e5be09e243483f29e1 100644 (file)
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_server.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/prometheus_server.sls
@@ -9,11 +9,9 @@ prometheus:
     component:
       - prometheus
       - alertmanager
-      - blackbox_exporter
+      - node_exporter
   pkg:
     use_upstream_repo: true
-    use_upstream_archive: true
-
     component:
       prometheus:
         config:
@@ -78,3 +76,20 @@ prometheus:
                   labels:
                     instance: database.__CLUSTER__
                     cluster: __CLUSTER__
+
+            # Nodes
+            - job_name: node
+              static_configs:
+                {% for node in [
+                  'controller',
+                  'keep0',
+                  'keep1',
+                  'keep',
+                  'workbench',
+                  'shell',
+                ] %}
+                - targets: [ "{{ node }}.__CLUSTER__.__DOMAIN__:9100" ]
+                  labels:
+                    instance: "{{ node }}.__CLUSTER__"
+                    cluster: __CLUSTER__
+                {% endfor %}

diff --git a/tools/salt-install/config_examples/multi_host/aws/states/postgresql_mtail.sls b/tools/salt-install/config_examples/multi_host/aws/states/prometheus_pg_exporter.sls
similarity index 91%
rename from tools/salt-install/config_examples/multi_host/aws/states/postgresql_mtail.sls
rename to tools/salt-install/config_examples/multi_host/aws/states/prometheus_pg_exporter.sls
index 6af01bbc240d1e469d060f0aa1db9b0b7b1d8e1e..dee2099fb2d8dff7cf8b390c75ac0e44d16c00d3 100644 (file)
--- a/tools/salt-install/config_examples/multi_host/aws/states/postgresql_mtail.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/states/prometheus_pg_exporter.sls
@@ -2,6 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+{%- set prometheus_pg_exporter = pillar.get('prometheus_pg_exporter', {'enabled': False}) %}
+
+{%- if prometheus_pg_exporter.enabled %}
 ### PACKAGES
 monitoring_required_pkgs:
   pkg.installed:
@@ -74,5 +77,6 @@ mtail_service:
     - require:
       - pkg: monitoring_required_pkgs
     - watch:
-      - file: /etc/mtail/postgresql.mtail
-      - file: /etc/default/mtail
+      - file: mtail_postgresql_conf
+      - file: mtail_etc_default
+{%- endif %}
\ No newline at end of file

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index b84367cf977b84b76cc3164b649aebd8c2b0a2a5..fd6a633c3dadeeeefe8ad82788c4aadbff426826 100755 (executable)
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -480,6 +480,7 @@ fi
 # Replace helper state files that differ from the formula's examples
 if [ -d "${SOURCE_STATES_DIR}" ]; then
   mkdir -p "${F_DIR}"/extra/extra
+  rm -f "${F_DIR}"/extra/extra/*
 
   for f in $(ls "${SOURCE_STATES_DIR}"/*); do
     sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
@@ -688,13 +689,17 @@ else
     grep -q "extra_custom_certs" ${P_DIR}/top.sls || echo "    - extra_custom_certs" >> ${P_DIR}/top.sls
   fi
 
+  # Prometheus state on all nodes due to the node exporter below
+  grep -q "\- prometheus$" ${S_DIR}/top.sls || echo "    - prometheus" >> ${S_DIR}/top.sls
+  # Prometheus node exporter pillar
+  grep -q "prometheus_node_exporter" ${P_DIR}/top.sls || echo "    - prometheus_node_exporter" >> ${P_DIR}/top.sls
+
   for R in ${ROLES}; do
     case "${R}" in
       "database")
         # States
         grep -q "\- postgres$" ${S_DIR}/top.sls || echo "    - postgres" >> ${S_DIR}/top.sls
-        grep -q "prometheus" ${S_DIR}/top.sls || echo "    - prometheus" >> ${S_DIR}/top.sls
-        grep -q "extra.postgresql_mtail" ${S_DIR}/top.sls || echo "    - extra.postgresql_mtail" >> ${S_DIR}/top.sls
+        grep -q "extra.prometheus_pg_exporter" ${S_DIR}/top.sls || echo "    - extra.prometheus_pg_exporter" >> ${S_DIR}/top.sls
         # Pillars
         grep -q "postgresql" ${P_DIR}/top.sls || echo "    - postgresql" >> ${P_DIR}/top.sls
         grep -q "prometheus_pg_exporter" ${P_DIR}/top.sls || echo "    - prometheus_pg_exporter" >> ${P_DIR}/top.sls
@@ -702,7 +707,6 @@ else
       "monitoring")
         ### States ###
         grep -q "nginx" ${S_DIR}/top.sls || echo "    - nginx" >> ${S_DIR}/top.sls
-        grep -q "prometheus" ${S_DIR}/top.sls || echo "    - prometheus" >> ${S_DIR}/top.sls
         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
           if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then