X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/4e110f379fe26b646c75cb3164f882567fdb17fd..HEAD:/tools/salt-install/provision.sh diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh index 3aa94aa585..edecc761c8 100755 --- a/tools/salt-install/provision.sh +++ b/tools/salt-install/provision.sh @@ -23,7 +23,8 @@ usage() { echo >&2 "${0} options:" echo >&2 " -d, --debug Run salt installation in debug mode" echo >&2 " -c , --config Path to the local.params config file" - echo >&2 " -t, --test Test installation running a CWL workflow" + echo >&2 " -t, --test Test cluster by running \`arvados-client diagnostics\`" + echo >&2 " and a simple workflow" echo >&2 " -r, --roles List of Arvados roles to apply to the host, comma separated" echo >&2 " Possible values are:" echo >&2 " balancer" @@ -172,10 +173,15 @@ apply_var_substitutions() { s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g; s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g; s#__INITIAL_USER__#${INITIAL_USER}#g; - s#__LE_AWS_REGION__#${LE_AWS_REGION}#g; - s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY}#g; - s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID}#g; + s#__LE_AWS_REGION__#${LE_AWS_REGION:-}#g; + s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY:-}#g; + s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID:-}#g; + s#__DATABASE_NAME__#${DATABASE_NAME}#g; + s#__DATABASE_USER__#${DATABASE_USER}#g; s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g; + s#__DATABASE_INT_IP__#${DATABASE_INT_IP:-}#g; + s#__DATABASE_EXTERNAL_SERVICE_HOST_OR_IP__#${DATABASE_EXTERNAL_SERVICE_HOST_OR_IP:-}#g; + s#__DATABASE_POSTGRESQL_VERSION__#${DATABASE_POSTGRESQL_VERSION}#g; s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g; s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g; s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g; @@ -196,13 +202,12 @@ apply_var_substitutions() { s#__SHELL_INT_IP__#${SHELL_INT_IP}#g; s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g; s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g; - s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g; - s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g; s#__SSL_KEY_ENCRYPTED__#${SSL_KEY_ENCRYPTED}#g; - s#__SSL_KEY_AWS_REGION__#${SSL_KEY_AWS_REGION}#g; + s#__SSL_KEY_AWS_REGION__#${SSL_KEY_AWS_REGION:-}#g; s#__SSL_KEY_AWS_SECRET_NAME__#${SSL_KEY_AWS_SECRET_NAME}#g; s#__CONTROLLER_MAX_WORKERS__#${CONTROLLER_MAX_WORKERS:-}#g; s#__CONTROLLER_MAX_QUEUED_REQUESTS__#${CONTROLLER_MAX_QUEUED_REQUESTS:-128}#g; + s#__CONTROLLER_MAX_GATEWAY_TUNNELS__#${CONTROLLER_MAX_GATEWAY_TUNNELS:-1000}#g; s#__MONITORING_USERNAME__#${MONITORING_USERNAME}#g; s#__MONITORING_EMAIL__#${MONITORING_EMAIL}#g; s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g; @@ -211,16 +216,19 @@ apply_var_substitutions() { s#__DISABLED_CONTROLLER__#${DISABLED_CONTROLLER}#g; s#__BALANCER_NODENAME__#${ROLE2NODES['balancer']:-}#g; s#__PROMETHEUS_NODENAME__#${ROLE2NODES['monitoring']:-}#g; - s#__CONTROLLER_NODES__#${ROLE2NODES['controller']}#g; + s#__PROMETHEUS_DATA_RETENTION_TIME__#${PROMETHEUS_DATA_RETENTION_TIME:-15d}#g; + s#__CONTROLLER_NODES__#${ROLE2NODES['controller']:-}#g; s#__NODELIST__#${NODELIST}#g; s#__DISPATCHER_INT_IP__#${DISPATCHER_INT_IP}#g; s#__KEEPBALANCE_INT_IP__#${KEEPBALANCE_INT_IP}#g; - s#__COMPUTE_AMI__#${COMPUTE_AMI}#g; - s#__COMPUTE_SG__#${COMPUTE_SG}#g; - s#__COMPUTE_SUBNET__#${COMPUTE_SUBNET}#g; - s#__COMPUTE_AWS_REGION__#${COMPUTE_AWS_REGION}#g; - s#__COMPUTE_USER__#${COMPUTE_USER}#g; - s#__KEEP_AWS_REGION__#${KEEP_AWS_REGION}#g" \ + s#__COMPUTE_AMI__#${COMPUTE_AMI:-}#g; + s#__COMPUTE_SG__#${COMPUTE_SG:-}#g; + s#__COMPUTE_SUBNET__#${COMPUTE_SUBNET:-}#g; + s#__COMPUTE_AWS_REGION__#${COMPUTE_AWS_REGION:-}#g; + s#__COMPUTE_USER__#${COMPUTE_USER:-}#g; + s#__KEEP_AWS_S3_BUCKET__#${KEEP_AWS_S3_BUCKET:-}#g; + s#__KEEP_AWS_IAM_ROLE__#${KEEP_AWS_IAM_ROLE:-}#g; + s#__KEEP_AWS_REGION__#${KEEP_AWS_REGION:-}#g" \ "${SRCFILE}" > "${DSTFILE}" } @@ -277,17 +285,15 @@ VERSION="latest" # We pin the salt version to avoid potential incompatibilities when a new # stable version is released. -SALT_VERSION="3004" +SALT_VERSION="3006" # Other formula versions we depend on -#POSTGRES_TAG="v0.44.0" -#POSTGRES_URL="https://github.com/saltstack-formulas/postgres-formula.git" -POSTGRES_TAG="0.45.0-bugfix327" -POSTGRES_URL="https://github.com/arvados/postgres-formula.git" +POSTGRES_TAG="7529300c287b1c288af0f494ca668c2217bd1c5d" +POSTGRES_URL="https://github.com/saltstack-formulas/postgres-formula.git" NGINX_TAG="v2.8.1" DOCKER_TAG="v2.4.2" -LOCALE_TAG="v0.3.4" -LETSENCRYPT_TAG="v2.1.0" +LOCALE_TAG="v0.3.5" +LETSENCRYPT_TAG="v3.2.0" LOGROTATE_TAG="v0.14.0" PROMETHEUS_TAG="v5.6.5" GRAFANA_TAG="v3.1.3" @@ -308,7 +314,6 @@ T_DIR="/tmp/cluster_tests" arguments ${@} declare -A NODES -declare -A ROLES declare -A ROLE2NODES declare NODELIST @@ -356,32 +361,34 @@ fi if [ "${DUMP_CONFIG}" = "yes" ]; then echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit" else - # Install a few dependency packages - # First, let's figure out the OS we're working on - OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d= |cut -f 2 -d \") - echo "Detected distro: ${OS_ID}" - - case ${OS_ID} in - "centos") - echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019" - sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux - setenforce permissive - yum install -y curl git jq - ;; - "debian"|"ubuntu") - # Wait 2 minutes for any apt locks to clear - # This option is supported from apt 1.9.1 and ignored in older apt versions. - # Cf. https://blog.sinjakli.co.uk/2021/10/25/waiting-for-apt-locks-without-the-hacky-bash-scripts/ - DEBIAN_FRONTEND=noninteractive apt -o DPkg::Lock::Timeout=120 update - DEBIAN_FRONTEND=noninteractive apt install -y curl git jq - ;; - esac + OS_IDS="$(. /etc/os-release && echo "${ID:-} ${ID_LIKE:-}")" + echo "Detected distro families: $OS_IDS" + + # Several of our formulas use the cron module, which requires the crontab + # command. We install systemd-cron to ensure we have that. + # The rest of these packages are required by the rest of the script. + for OS_ID in $OS_IDS; do + case "$OS_ID" in + rhel) + echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019" + sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux + setenforce permissive + yum install -y curl git jq systemd-cron + break + ;; + debian) + DEBIAN_FRONTEND=noninteractive apt -o DPkg::Lock::Timeout=120 update + DEBIAN_FRONTEND=noninteractive apt install -y curl git jq systemd-cron + break + ;; + esac + done if which salt-call; then echo "Salt already installed" else curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh - sh /tmp/bootstrap_salt.sh -XdfP -x python3 old-stable ${SALT_VERSION} + sh /tmp/bootstrap_salt.sh -XdfP -x python3 stable ${SALT_VERSION} /bin/systemctl stop salt-minion.service /bin/systemctl disable salt-minion.service fi @@ -424,7 +431,7 @@ test -d nginx && ( cd nginx && git fetch ) \ echo "...postgres" test -d postgres && ( cd postgres && git fetch ) \ || git clone --quiet ${POSTGRES_URL} ${F_DIR}/postgres -( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" ) +( cd postgres && git checkout --quiet "${POSTGRES_TAG}" ) echo "...prometheus" test -d prometheus && ( cd prometheus && git fetch ) \ @@ -450,8 +457,8 @@ echo "...arvados" test -d arvados || git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados # If we want to try a specific branch of the formula -if [ "x${BRANCH:-}" != "x" ]; then - ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" ) +if [[ ! -z "${BRANCH:-}" && "x${BRANCH}" != "xmain" ]]; then + ( cd ${F_DIR}/arvados && git fetch && git checkout --quiet "${BRANCH}" || git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" ) elif [ "x${ARVADOS_TAG:-}" != "x" ]; then ( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" ) fi @@ -514,7 +521,7 @@ fi # Replace helper state files that differ from the formula's examples if [ -d "${SOURCE_STATES_DIR}" ]; then mkdir -p "${F_DIR}"/extra/extra - rm -f "${F_DIR}"/extra/extra/* + rm -rf "${F_DIR}"/extra/extra/* for f in $(ls "${SOURCE_STATES_DIR}"/*); do apply_var_substitutions "${f}" "${F_DIR}/extra/extra"/$(basename "${f}") @@ -567,7 +574,7 @@ fi # If we want specific roles for a node, just add the desired states # and its dependencies -if [ -z "${ROLES}" ]; then +if [ -z "${ROLES:-}" ]; then # States echo " - nginx.passenger" >> ${STATES_TOP} if [ "${SSL_MODE}" = "lets-encrypt" ]; then @@ -593,14 +600,27 @@ if [ -z "${ROLES}" ]; then echo " - postgres" >> ${STATES_TOP} echo " - logrotate" >> ${STATES_TOP} echo " - docker.software" >> ${STATES_TOP} - echo " - arvados" >> ${STATES_TOP} + echo " - arvados.repo" >> ${STATES_TOP} + echo " - arvados.config" >> ${STATES_TOP} + echo " - arvados.ruby" >> ${STATES_TOP} + echo " - arvados.api" >> ${STATES_TOP} + echo " - arvados.controller" >> ${STATES_TOP} + echo " - arvados.keepstore" >> ${STATES_TOP} + echo " - arvados.websocket" >> ${STATES_TOP} + echo " - arvados.keepweb" >> ${STATES_TOP} + echo " - arvados.workbench2" >> ${STATES_TOP} + echo " - arvados.keepproxy" >> ${STATES_TOP} + echo " - arvados.shell" >> ${STATES_TOP} + echo " - arvados.dispatcher" >> ${STATES_TOP} echo " - extra.shell_sudo_passwordless" >> ${STATES_TOP} echo " - extra.shell_cron_add_login_sync" >> ${STATES_TOP} echo " - extra.passenger_rvm" >> ${STATES_TOP} + echo " - extra.workbench1_uninstall" >> ${STATES_TOP} # Pillars echo " - docker" >> ${PILLARS_TOP} echo " - nginx_api_configuration" >> ${PILLARS_TOP} + echo " - logrotate" >> ${PILLARS_TOP} echo " - logrotate_api" >> ${PILLARS_TOP} echo " - nginx_controller_configuration" >> ${PILLARS_TOP} echo " - nginx_keepproxy_configuration" >> ${PILLARS_TOP} @@ -670,7 +690,7 @@ if [ -z "${ROLES}" ]; then grep -q ${CERT_NAME} ${P_DIR}/extra_custom_certs.sls || echo " - ${CERT_NAME}" >> ${P_DIR}/extra_custom_certs.sls # As the pillar differs whether we use LE or custom certs, we need to do a final edition on them - sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${CERT_NAME}.pem/g; + sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_${CERT_NAME}_cert_file_copy/g; s#__CERT_PEM__#/etc/nginx/ssl/arvados-${CERT_NAME}.pem#g; s#__CERT_KEY__#/etc/nginx/ssl/arvados-${CERT_NAME}.key#g" \ ${P_DIR}/nginx_${c}_configuration.sls @@ -698,7 +718,7 @@ else # Prometheus node exporter pillar grep -q "prometheus_node_exporter" ${PILLARS_TOP} || echo " - prometheus_node_exporter" >> ${PILLARS_TOP} - for R in ${ROLES}; do + for R in ${ROLES:-}; do case "${R}" in "database") # States @@ -716,7 +736,9 @@ else # "ArvadosPromDataSource" is the hardcoded UID for Prometheus' datasource # in Grafana. for f in $(ls "${GRAFANA_DASHBOARDS_DIR}"/*.json); do - sed 's#${DS_PROMETHEUS}#ArvadosPromDataSource#g' \ + sed "s#__TLS_EXPIRATION_YELLOW__#${TLS_EXPIRATION_YELLOW}#g; + s#__TLS_EXPIRATION_GREEN__#${TLS_EXPIRATION_GREEN}#g; + s#\${DS_PROMETHEUS}#ArvadosPromDataSource#g" \ "${f}" > "${GRAFANA_DASHBOARDS_DEST_DIR}"/$(basename "${f}") done @@ -745,6 +767,7 @@ else for SVC in grafana prometheus; do grep -q "nginx_${SVC}_configuration" ${PILLARS_TOP} || echo " - nginx_${SVC}_configuration" >> ${PILLARS_TOP} done + grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP} if [ "${SSL_MODE}" = "lets-encrypt" ]; then grep -q "letsencrypt" ${PILLARS_TOP} || echo " - letsencrypt" >> ${PILLARS_TOP} for SVC in grafana prometheus; do @@ -760,7 +783,7 @@ else elif [ "${SSL_MODE}" = "bring-your-own" ]; then grep -q "ssl_key_encrypted" ${PILLARS_TOP} || echo " - ssl_key_encrypted" >> ${PILLARS_TOP} for SVC in grafana prometheus; do - sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${SVC}.pem/g; + sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_${SVC}_cert_file_copy/g; s#__CERT_PEM__#/etc/nginx/ssl/arvados-${SVC}.pem#g; s#__CERT_KEY__#/etc/nginx/ssl/arvados-${SVC}.key#g" \ ${P_DIR}/nginx_${SVC}_configuration.sls @@ -798,7 +821,7 @@ else fi elif [ "${SSL_MODE}" = "bring-your-own" ]; then grep -q "ssl_key_encrypted" ${PILLARS_TOP} || echo " - ssl_key_encrypted" >> ${PILLARS_TOP} - sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g; + sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_${R}_cert_file_copy/g; s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g; s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \ ${P_DIR}/nginx_${R}_configuration.sls @@ -814,6 +837,7 @@ else echo " - nginx.passenger" >> ${STATES_TOP} fi echo " - extra.passenger_rvm" >> ${STATES_TOP} + grep -q "^ - postgres\\.client$" ${STATES_TOP} || echo " - postgres.client" >> ${STATES_TOP} ### If we don't install and run LE before arvados-api-server, it fails and breaks everything ### after it. So we add this here as we are, after all, sharing the host for api and controller @@ -832,10 +856,12 @@ else grep -q "arvados.controller" ${STATES_TOP} || echo " - arvados.controller" >> ${STATES_TOP} ### Pillars ### + grep -q "logrotate" ${PILLARS_TOP} || echo " - logrotate" >> ${PILLARS_TOP} grep -q "logrotate_api" ${PILLARS_TOP} || echo " - logrotate_api" >> ${PILLARS_TOP} grep -q "aws_credentials" ${PILLARS_TOP} || echo " - aws_credentials" >> ${PILLARS_TOP} grep -q "postgresql" ${PILLARS_TOP} || echo " - postgresql" >> ${PILLARS_TOP} grep -q "nginx_passenger" ${PILLARS_TOP} || echo " - nginx_passenger" >> ${PILLARS_TOP} + grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP} grep -q "nginx_api_configuration" ${PILLARS_TOP} || echo " - nginx_api_configuration" >> ${PILLARS_TOP} grep -q "nginx_controller_configuration" ${PILLARS_TOP} || echo " - nginx_controller_configuration" >> ${PILLARS_TOP} @@ -853,7 +879,7 @@ else ${P_DIR}/nginx_${R}_configuration.sls else grep -q "ssl_key_encrypted" ${PILLARS_TOP} || echo " - ssl_key_encrypted" >> ${PILLARS_TOP} - sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g; + sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_${R}_cert_file_copy/g; s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g; s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \ ${P_DIR}/nginx_${R}_configuration.sls @@ -866,17 +892,7 @@ else ;; "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy") ### States ### - if [ "${R}" = "workbench" ]; then - grep -q " - logrotate" ${STATES_TOP} || echo " - logrotate" >> ${STATES_TOP} - NGINX_INSTALL_SOURCE="install_from_phusionpassenger" - if grep -q " - nginx$" ${STATES_TOP}; then - sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${STATES_TOP} - else - echo " - nginx.passenger" >> ${STATES_TOP} - fi - else - grep -q "\- nginx$" ${STATES_TOP} || echo " - nginx" >> ${STATES_TOP} - fi + grep -q "\- nginx$" ${STATES_TOP} || echo " - nginx" >> ${STATES_TOP} if [ "${SSL_MODE}" = "lets-encrypt" ]; then if [ "x${USE_LETSENCRYPT_ROUTE53:-}" = "xyes" ]; then @@ -898,16 +914,18 @@ else fi # webshell role is just a nginx vhost, so it has no state - if [ "${R}" != "webshell" ]; then + # workbench role is deprecated since 2.7.0 + if [[ "${R}" != "webshell" && "${R}" != "workbench" ]]; then grep -q "arvados.${R}" ${STATES_TOP} || echo " - arvados.${R}" >> ${STATES_TOP} fi + # Make sure wb1's package get uninstalled + if [[ "${R}" == "workbench" ]]; then + grep -q "workbench1_uninstall" ${STATES_TOP} || echo " - extra.workbench1_uninstall" >> ${STATES_TOP} + fi ### Pillars ### - if [ "${R}" = "workbench" ]; then - grep -q "logrotate_wb1" ${PILLARS_TOP} || echo " - logrotate_wb1" >> ${PILLARS_TOP} - fi - grep -q "nginx_passenger" ${PILLARS_TOP} || echo " - nginx_passenger" >> ${PILLARS_TOP} grep -q "nginx_${R}_configuration" ${PILLARS_TOP} || echo " - nginx_${R}_configuration" >> ${PILLARS_TOP} + grep -q "nginx_snippets" ${PILLARS_TOP} || echo " - nginx_snippets" >> ${PILLARS_TOP} # Special case for keepweb if [ ${R} = "keepweb" ]; then grep -q "nginx_download_configuration" ${PILLARS_TOP} || echo " - nginx_download_configuration" >> ${PILLARS_TOP} @@ -942,14 +960,14 @@ else # Special case for keepweb if [ ${R} = "keepweb" ]; then for kwsub in download collections; do - sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${kwsub}.pem/g; + sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_${kwsub}_cert_file_copy/g; s#__CERT_PEM__#/etc/nginx/ssl/arvados-${kwsub}.pem#g; s#__CERT_KEY__#/etc/nginx/ssl/arvados-${kwsub}.key#g" \ ${P_DIR}/nginx_${kwsub}_configuration.sls grep -q ${kwsub} ${P_DIR}/extra_custom_certs.sls || echo " - ${kwsub}" >> ${P_DIR}/extra_custom_certs.sls done else - sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g; + sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_${R}_cert_file_copy/g; s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g; s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \ ${P_DIR}/nginx_${R}_configuration.sls