echo >&2 " keepbalance"
echo >&2 " keepstore"
echo >&2 " keepweb"
+ echo >&2 " monitoring"
echo >&2 " shell"
echo >&2 " webshell"
echo >&2 " websocket"
for i in ${2//,/ }
do
# Verify the role exists
- if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
+ if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher,monitoring" == *"$i"* ]]; then
echo "The role '${i}' is not a valid role"
usage
exit 1
cert_dir=${1}
cert_name=${2}
- mkdir -p /srv/salt/certs
+ mkdir -p --mode=0700 /srv/salt/certs
if [ -f ${cert_dir}/${cert_name}.crt ]; then
- cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
+ install --mode=0600 ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
else
echo "${cert_dir}/${cert_name}.crt does not exist. Exiting"
exit 1
fi
if [ -f ${cert_dir}/${cert_name}.key ]; then
- cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
+ install --mode=0600 ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
else
echo "${cert_dir}/${cert_name}.key does not exist. Exiting"
exit 1
USE_LETSENCRYPT_ROUTE53="no"
CUSTOM_CERTS_DIR="${SCRIPT_DIR}/local_config_dir/certs"
+GRAFANA_DASHBOARDS_DIR="${SCRIPT_DIR}/local_config_dir/dashboards"
+
## These are ARVADOS-related parameters
# For a stable release, change RELEASE "production" and VERSION to the
# package version (including the iteration, e.g. X.Y.Z-1) of the
SALT_VERSION="3004"
# Other formula versions we depend on
-POSTGRES_TAG="v0.44.0"
+#POSTGRES_TAG="v0.44.0"
+#POSTGRES_URL="https://github.com/saltstack-formulas/postgres-formula.git"
+POSTGRES_TAG="0.45.0-bugfix327"
+POSTGRES_URL="https://github.com/arvados/postgres-formula.git"
NGINX_TAG="v2.8.1"
DOCKER_TAG="v2.4.2"
LOCALE_TAG="v0.3.4"
LETSENCRYPT_TAG="v2.1.0"
LOGROTATE_TAG="v0.14.0"
+PROMETHEUS_TAG="v5.6.5"
+GRAFANA_TAG="v3.1.3"
# Salt's dir
DUMP_SALT_CONFIG_DIR=""
USE_SINGLE_HOSTNAME="no"
# We set this variable, anyway, so sed lines do not fail and we don't need to add more
# conditionals
- HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
+ HOSTNAME_EXT="${DOMAIN}"
fi
if [ "${DUMP_CONFIG}" = "yes" ]; then
echo "...postgres"
test -d postgres && ( cd postgres && git fetch ) \
- || git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
+ || git clone --quiet ${POSTGRES_URL} ${F_DIR}/postgres
( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" )
+echo "...prometheus"
+test -d prometheus && ( cd prometheus && git fetch ) \
+ || git clone --quiet https://github.com/saltstack-formulas/prometheus-formula.git ${F_DIR}/prometheus
+( cd prometheus && git checkout --quiet tags/"${PROMETHEUS_TAG}" )
+
+echo "...grafana"
+test -d grafana && ( cd grafana && git fetch ) \
+ || git clone --quiet https://github.com/saltstack-formulas/grafana-formula.git ${F_DIR}/grafana
+( cd grafana && git checkout --quiet "${GRAFANA_TAG}" )
+
echo "...letsencrypt"
test -d letsencrypt && ( cd letsencrypt && git fetch ) \
|| git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
if [ "x${VAGRANT}" = "xyes" ]; then
EXTRA_STATES_DIR="/home/vagrant/${CONFIG_DIR}/states"
SOURCE_PILLARS_DIR="/home/vagrant/${CONFIG_DIR}/pillars"
+ SOURCE_TOFS_DIR="/home/vagrant/${CONFIG_DIR}/tofs"
SOURCE_TESTS_DIR="/home/vagrant/${TESTS_DIR}"
else
EXTRA_STATES_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/states"
SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
+ SOURCE_TOFS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/tofs"
SOURCE_TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
fi
s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
- s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
- s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
+ s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g;
+ s#__SSL_KEY_ENCRYPTED__#${SSL_KEY_ENCRYPTED}#g;
+ s#__SSL_KEY_AWS_REGION__#${SSL_KEY_AWS_REGION}#g;
+ s#__SSL_KEY_AWS_SECRET_NAME__#${SSL_KEY_AWS_SECRET_NAME}#g;
+ s#__CONTROLLER_NGINX_WORKERS__#${CONTROLLER_NGINX_WORKERS}#g;
+ s#__CONTROLLER_MAX_CONCURRENT_REQUESTS__#${CONTROLLER_MAX_CONCURRENT_REQUESTS}#g;
+ s#__MONITORING_USERNAME__#${MONITORING_USERNAME}#g;
+ s#__MONITORING_EMAIL__#${MONITORING_EMAIL}#g;
+ s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g" \
"${f}" > "${P_DIR}"/$(basename "${f}")
done
# Replace helper state files that differ from the formula's examples
if [ -d "${SOURCE_STATES_DIR}" ]; then
mkdir -p "${F_DIR}"/extra/extra
+ rm -f "${F_DIR}"/extra/extra/*
for f in $(ls "${SOURCE_STATES_DIR}"/*); do
sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
- s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
- s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
+ s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g;
+ s#__SSL_KEY_ENCRYPTED__#${SSL_KEY_ENCRYPTED}#g;
+ s#__SSL_KEY_AWS_REGION__#${SSL_KEY_AWS_REGION}#g;
+ s#__SSL_KEY_AWS_SECRET_NAME__#${SSL_KEY_AWS_SECRET_NAME}#g;
+ s#__CONTROLLER_NGINX_WORKERS__#${CONTROLLER_NGINX_WORKERS}#g;
+ s#__CONTROLLER_MAX_CONCURRENT_REQUESTS__#${CONTROLLER_MAX_CONCURRENT_REQUESTS}#g;
+ s#__MONITORING_USERNAME__#${MONITORING_USERNAME}#g;
+ s#__MONITORING_EMAIL__#${MONITORING_EMAIL}#g;
+ s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g" \
"${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
done
fi
# As we need to separate both states and pillars in case we want specific
# roles, we iterate on both at the same time
+# Formula template overrides (TOFS)
+# See: https://template-formula.readthedocs.io/en/latest/TOFS_pattern.html#template-override
+if [ -d ${SOURCE_TOFS_DIR} ]; then
+ find ${SOURCE_TOFS_DIR} -mindepth 1 -maxdepth 1 -type d -exec cp -r "{}" ${S_DIR} \;
+fi
+
# States
cat > ${S_DIR}/top.sls << EOFTSLS
base:
if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - extra.aws_credentials" >> ${S_DIR}/top.sls
fi
- grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
else
- mkdir -p /srv/salt/certs
+ mkdir -p --mode=0700 /srv/salt/certs
if [ "${SSL_MODE}" = "bring-your-own" ]; then
# Copy certs to formula extra/files
- cp -rv ${CUSTOM_CERTS_DIR}/* /srv/salt/certs/
+ install --mode=0600 ${CUSTOM_CERTS_DIR}/* /srv/salt/certs/
# We add the custom_certs state
- grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+ grep -q "custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+ if [ "${SSL_KEY_ENCRYPTED}" = "yes" ]; then
+ grep -q "ssl_key_encrypted" ${S_DIR}/top.sls || echo " - extra.ssl_key_encrypted" >> ${S_DIR}/top.sls
+ fi
fi
# In self-signed mode, the certificate files will be created and put in the
# destination directory by the snakeoil_certs.sls state file
CERT_NAME=${HOSTNAME_EXT}
else
# We are in a multiple-hostnames env
- CERT_NAME=${c}.${CLUSTER}.${DOMAIN}
+ CERT_NAME=${c}.${DOMAIN}
fi
# As the pillar differs whether we use LE or custom certs, we need to do a final edition on them
echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
- for c in controller websocket workbench workbench2 webshell keepweb keepproxy shell; do
+ for c in controller websocket workbench workbench2 webshell keepweb keepproxy; do
# Are we in a single-host-single-hostname env?
if [ "${USE_SINGLE_HOSTNAME}" = "yes" ]; then
# Are we in a single-host-single-hostname env?
echo " - arvados.repo" >> ${S_DIR}/top.sls
# We add the extra_custom_certs state
grep -q "extra.custom_certs" ${S_DIR}/top.sls || echo " - extra.custom_certs" >> ${S_DIR}/top.sls
+ if [ "${SSL_KEY_ENCRYPTED}" = "yes" ]; then
+ grep -q "ssl_key_encrypted" ${S_DIR}/top.sls || echo " - extra.ssl_key_encrypted" >> ${S_DIR}/top.sls
+ fi
# And we add the basic part for the certs pillar
if [ "${SSL_MODE}" != "lets-encrypt" ]; then
grep -q "extra_custom_certs" ${P_DIR}/top.sls || echo " - extra_custom_certs" >> ${P_DIR}/top.sls
fi
+ # Prometheus state on all nodes due to the node exporter below
+ grep -q "\- prometheus$" ${S_DIR}/top.sls || echo " - prometheus" >> ${S_DIR}/top.sls
+ # Prometheus node exporter pillar
+ grep -q "prometheus_node_exporter" ${P_DIR}/top.sls || echo " - prometheus_node_exporter" >> ${P_DIR}/top.sls
+
for R in ${ROLES}; do
case "${R}" in
"database")
# States
- echo " - postgres" >> ${S_DIR}/top.sls
+ grep -q "\- postgres$" ${S_DIR}/top.sls || echo " - postgres" >> ${S_DIR}/top.sls
+ grep -q "extra.prometheus_pg_exporter" ${S_DIR}/top.sls || echo " - extra.prometheus_pg_exporter" >> ${S_DIR}/top.sls
# Pillars
- echo ' - postgresql' >> ${P_DIR}/top.sls
+ grep -q "postgresql" ${P_DIR}/top.sls || echo " - postgresql" >> ${P_DIR}/top.sls
+ grep -q "prometheus_pg_exporter" ${P_DIR}/top.sls || echo " - prometheus_pg_exporter" >> ${P_DIR}/top.sls
+ ;;
+ "monitoring")
+ ### Support files ###
+ GRAFANA_DASHBOARDS_DEST_DIR=/srv/salt/dashboards
+ mkdir -p "${GRAFANA_DASHBOARDS_DEST_DIR}"
+ rm -f "${GRAFANA_DASHBOARDS_DEST_DIR}"/*
+ # "ArvadosPromDataSource" is the hardcoded UID for Prometheus' datasource
+ # in Grafana.
+ for f in $(ls "${GRAFANA_DASHBOARDS_DIR}"/*.json); do
+ sed 's#${DS_PROMETHEUS}#ArvadosPromDataSource#g' \
+ "${f}" > "${GRAFANA_DASHBOARDS_DEST_DIR}"/$(basename "${f}")
+ done
+
+ ### States ###
+ grep -q "\- nginx$" ${S_DIR}/top.sls || echo " - nginx" >> ${S_DIR}/top.sls
+ grep -q "extra.nginx_prometheus_configuration" ${S_DIR}/top.sls || echo " - extra.nginx_prometheus_configuration" >> ${S_DIR}/top.sls
+
+ grep -q "\- grafana$" ${S_DIR}/top.sls || echo " - grafana" >> ${S_DIR}/top.sls
+ grep -q "extra.grafana_datasource" ${S_DIR}/top.sls || echo " - extra.grafana_datasource" >> ${S_DIR}/top.sls
+ grep -q "extra.grafana_dashboards" ${S_DIR}/top.sls || echo " - extra.grafana_dashboards" >> ${S_DIR}/top.sls
+ grep -q "extra.grafana_admin_user" ${S_DIR}/top.sls || echo " - extra.grafana_admin_user" >> ${S_DIR}/top.sls
+
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ grep -q "letsencrypt" ${S_DIR}/top.sls || echo " - letsencrypt" >> ${S_DIR}/top.sls
+ if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
+ grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls
+ fi
+ elif [ "${SSL_MODE}" = "bring-your-own" ]; then
+ for SVC in grafana prometheus; do
+ copy_custom_cert ${CUSTOM_CERTS_DIR} ${SVC}
+ done
+ fi
+ ### Pillars ###
+ grep -q "prometheus_server" ${P_DIR}/top.sls || echo " - prometheus_server" >> ${P_DIR}/top.sls
+ grep -q "grafana" ${P_DIR}/top.sls || echo " - grafana" >> ${P_DIR}/top.sls
+ for SVC in grafana prometheus; do
+ grep -q "nginx_${SVC}_configuration" ${P_DIR}/top.sls || echo " - nginx_${SVC}_configuration" >> ${P_DIR}/top.sls
+ done
+ if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+ grep -q "letsencrypt" ${P_DIR}/top.sls || echo " - letsencrypt" >> ${P_DIR}/top.sls
+ for SVC in grafana prometheus; do
+ grep -q "letsencrypt_${SVC}_configuration" ${P_DIR}/top.sls || echo " - letsencrypt_${SVC}_configuration" >> ${P_DIR}/top.sls
+ sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${SVC}.${DOMAIN}*/g;
+ s#__CERT_PEM__#/etc/letsencrypt/live/${SVC}.${DOMAIN}/fullchain.pem#g;
+ s#__CERT_KEY__#/etc/letsencrypt/live/${SVC}.${DOMAIN}/privkey.pem#g" \
+ ${P_DIR}/nginx_${SVC}_configuration.sls
+ done
+ if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
+ grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls
+ fi
+ elif [ "${SSL_MODE}" = "bring-your-own" ]; then
+ grep -q "ssl_key_encrypted" ${P_DIR}/top.sls || echo " - ssl_key_encrypted" >> ${P_DIR}/top.sls
+ for SVC in grafana prometheus; do
+ sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${SVC}.pem/g;
+ s#__CERT_PEM__#/etc/nginx/ssl/arvados-${SVC}.pem#g;
+ s#__CERT_KEY__#/etc/nginx/ssl/arvados-${SVC}.key#g" \
+ ${P_DIR}/nginx_${SVC}_configuration.sls
+ grep -q ${SVC} ${P_DIR}/extra_custom_certs.sls || echo " - ${SVC}" >> ${P_DIR}/extra_custom_certs.sls
+ done
+ fi
;;
"api")
# States
echo " - nginx.passenger" >> ${S_DIR}/top.sls
fi
else
- grep -q "nginx" ${S_DIR}/top.sls || echo " - nginx" >> ${S_DIR}/top.sls
+ grep -q "\- nginx$" ${S_DIR}/top.sls || echo " - nginx" >> ${S_DIR}/top.sls
fi
if [ "${SSL_MODE}" = "lets-encrypt" ]; then
if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
# Special case for keepweb
if [ ${R} = "keepweb" ]; then
for kwsub in download collections; do
- sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${kwsub}.${CLUSTER}.${DOMAIN}*/g;
- s#__CERT_PEM__#/etc/letsencrypt/live/${kwsub}.${CLUSTER}.${DOMAIN}/fullchain.pem#g;
- s#__CERT_KEY__#/etc/letsencrypt/live/${kwsub}.${CLUSTER}.${DOMAIN}/privkey.pem#g" \
+ sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${kwsub}.${DOMAIN}*/g;
+ s#__CERT_PEM__#/etc/letsencrypt/live/${kwsub}.${DOMAIN}/fullchain.pem#g;
+ s#__CERT_KEY__#/etc/letsencrypt/live/${kwsub}.${DOMAIN}/privkey.pem#g" \
${P_DIR}/nginx_${kwsub}_configuration.sls
done
else
- sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${R}.${CLUSTER}.${DOMAIN}*/g;
- s#__CERT_PEM__#/etc/letsencrypt/live/${R}.${CLUSTER}.${DOMAIN}/fullchain.pem#g;
- s#__CERT_KEY__#/etc/letsencrypt/live/${R}.${CLUSTER}.${DOMAIN}/privkey.pem#g" \
+ sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${R}.${DOMAIN}*/g;
+ s#__CERT_PEM__#/etc/letsencrypt/live/${R}.${DOMAIN}/fullchain.pem#g;
+ s#__CERT_KEY__#/etc/letsencrypt/live/${R}.${DOMAIN}/privkey.pem#g" \
${P_DIR}/nginx_${R}_configuration.sls
fi
else
+ grep -q "ssl_key_encrypted" ${P_DIR}/top.sls || echo " - ssl_key_encrypted" >> ${P_DIR}/top.sls
# As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
# Special case for keepweb
if [ ${R} = "keepweb" ]; then
fi
# Now run the install
-salt-call --local state.apply -l ${LOG_LEVEL}
+salt-call --state-output=mixed --local state.apply -l ${LOG_LEVEL}
# Finally, make sure that /etc/hosts is not overwritten on reboot
if [ -d /etc/cloud/cloud.cfg.d ]; then
fi
# Leave a copy of the Arvados CA so the user can copy it where it's required
-if [ "$DEV_MODE" = "yes" ]; then
- echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
- # If running in a vagrant VM, also add default user to docker group
+if [ "${SSL_MODE}" = "self-signed" ]; then
+ echo "Copying the Arvados CA certificate '${DOMAIN}-arvados-snakeoil-ca.crt' to the installer dir, so you can import it"
if [ "x${VAGRANT}" = "xyes" ]; then
- cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
+ cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${DOMAIN}-arvados-snakeoil-ca.pem
+ else
+ cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${DOMAIN}-arvados-snakeoil-ca.crt
+ fi
+fi
+if [ "x${VAGRANT}" = "xyes" ]; then
+ # If running in a vagrant VM, also add default user to docker group
echo "Adding the vagrant user to the docker group"
usermod -a -G docker vagrant
- else
- cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
- fi
fi
# Test that the installation finished correctly