From: Javier Bértoli Date: Thu, 17 Mar 2022 18:19:06 +0000 (-0300) Subject: Merge branch '18681-install-passenger-only-when-required' X-Git-Tag: 2.4.0~41 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/9a11b502e406202db58fb6751f3448aa30288062?hp=caf63ceb7b163ee37756b07a714dbc32944f9567 Merge branch '18681-install-passenger-only-when-required' closes #18681 Arvados-DCO-1.1-Signed-off-by: Javier Bértoli --- diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile index a8dd345343..1573b6862b 100644 --- a/tools/salt-install/Vagrantfile +++ b/tools/salt-install/Vagrantfile @@ -41,6 +41,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| /vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames" # s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;' \ + arv.vm.provision "shell", + inline: "cp -vr /tmp/local.params.single_host_multiple_hostnames /tmp/local.params.single_host_multiple_hostnames.falla; + cp -vr /vagrant/centos7-local.params.single_host_single_hostname-f258b604f831bb3bd7fab506c670b975ae8e4118 /tmp/local.params.single_host_multiple_hostnames" arv.vm.provision "shell", path: "provision.sh", args: [ @@ -54,7 +57,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| # A single_host single_hostname example config.vm.define "arvados-sh-sn" do |arv| - arv.vm.box = "bento/debian-11" + #arv.vm.box = "bento/centos-7" + arv.vm.box = "bento/ubuntu-20.04" arv.vm.hostname = "zeppo" # CPU/RAM config.vm.provider :virtualbox do |v| @@ -80,12 +84,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| arv.vm.provision "shell", inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir; cp -vr /vagrant/tests /home/vagrant/tests; - sed 's#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"zeppo.local\"#g; - 's#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g; - s#cluster_fixme_or_this_wont_work#zeppo#g; - s#domain_fixme_or_this_wont_work#local#g;' \ + sed 's#cluster_fixme_or_this_wont_work#cnts7#g; + s#domain_fixme_or_this_wont_work#local#g; + s#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"cnts7.local\"#g; + s#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g; + s#RELEASE=\"production\"#RELEASE=\"development\"#g; + s/# BRANCH=\"main\"/BRANCH=\"main\"/g; + s/# VERSION=.*$/VERSION=\"latest\"/g' \ /vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname" - # s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g; + arv.vm.provision "shell", path: "provision.sh", args: [ diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls index d9644e5645..fbd42bd7a3 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls @@ -15,7 +15,7 @@ ### NGINX nginx: - install_from_phusionpassenger: true + __NGINX_INSTALL_SOURCE__: true lookup: passenger_package: {{ passenger_pkg }} ### PASSENGER @@ -25,11 +25,15 @@ nginx: ### SERVER server: config: + # As we now differentiate where passenger is required or not, we need to + # load this module conditionally, so we add this conditional just to use + # the same pillar file + {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %} # This is required to get the passenger module loaded # In Debian it can be done with this # include: 'modules-enabled/*.conf' load_module: {{ passenger_mod }} - + {% endif %} worker_processes: 4 ### SNIPPETS diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls index a0da9a1c05..e06ddd041c 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls @@ -6,7 +6,7 @@ ### POSTGRESQL postgres: use_upstream_repo: true - version: '11' + version: '12' postgresconf: |- listen_addresses = '*' # listen on all interfaces acls: diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls index 886aab8764..dbf21c2651 100644 --- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls +++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls @@ -15,7 +15,7 @@ ### NGINX nginx: - install_from_phusionpassenger: true + __NGINX_INSTALL_SOURCE__: true lookup: passenger_package: {{ passenger_pkg }} ### PASSENGER @@ -25,11 +25,15 @@ nginx: ### SERVER server: config: + # As we now differentiate where passenger is required or not, we need to + # load this module conditionally, so we add this conditional just to use + # the same pillar file + {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %} # This is required to get the passenger module loaded # In Debian it can be done with this # include: 'modules-enabled/*.conf' load_module: {{ passenger_mod }} - + {% endif %} worker_processes: 4 ### SNIPPETS diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls index bce51f29e0..c25720c60a 100644 --- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls +++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls @@ -15,7 +15,7 @@ ### NGINX nginx: - install_from_phusionpassenger: true + __NGINX_INSTALL_SOURCE__: true lookup: passenger_package: {{ passenger_pkg }} ### PASSENGER @@ -25,11 +25,15 @@ nginx: ### SERVER server: config: + # As we now differentiate where passenger is required or not, we need to + # load this module conditionally, so we add this conditional just to use + # the same pillar file + {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %} # This is required to get the passenger module loaded # In Debian it can be done with this # include: 'modules-enabled/*.conf' load_module: {{ passenger_mod }} - + {% endif %} worker_processes: 4 ### SNIPPETS diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts index 221e7b35eb..32d1f8bb96 100644 --- a/tools/salt-install/local.params.example.multiple_hosts +++ b/tools/salt-install/local.params.example.multiple_hosts @@ -119,8 +119,8 @@ RELEASE="production" # Formulas versions # ARVADOS_TAG="2.2.0" -# POSTGRES_TAG="v0.43.0" -# NGINX_TAG="v2.8.0" -# DOCKER_TAG="v2.0.7" +# POSTGRES_TAG="v0.44.0" +# NGINX_TAG="v2.8.1" +# DOCKER_TAG="v2.4.2" # LOCALE_TAG="v0.3.4" # LETSENCRYPT_TAG="v2.1.0" diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames index 77015248c1..d6bfb102e9 100644 --- a/tools/salt-install/local.params.example.single_host_multiple_hostnames +++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames @@ -70,8 +70,8 @@ RELEASE="production" # Formulas versions # ARVADOS_TAG="2.2.0" -# POSTGRES_TAG="v0.43.0" -# NGINX_TAG="v2.8.0" -# DOCKER_TAG="v2.0.7" +# POSTGRES_TAG="v0.44.0" +# NGINX_TAG="v2.8.1" +# DOCKER_TAG="v2.4.2" # LOCALE_TAG="v0.3.4" # LETSENCRYPT_TAG="v2.1.0" diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname index a1cd491bc2..b6c7e5f7a5 100644 --- a/tools/salt-install/local.params.example.single_host_single_hostname +++ b/tools/salt-install/local.params.example.single_host_single_hostname @@ -79,7 +79,7 @@ RELEASE="production" # Formulas versions # ARVADOS_TAG="2.2.0" # POSTGRES_TAG="v0.44.0" -# NGINX_TAG="v2.8.0" -# DOCKER_TAG="v2.4.0" +# NGINX_TAG="v2.8.1" +# DOCKER_TAG="v2.4.2" # LOCALE_TAG="v0.3.4" # LETSENCRYPT_TAG="v2.1.0" diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh index 2d33cac57b..0f3c9a1411 100755 --- a/tools/salt-install/provision.sh +++ b/tools/salt-install/provision.sh @@ -165,6 +165,8 @@ LOG_LEVEL="info" CONTROLLER_EXT_SSL_PORT=443 TESTS_DIR="tests" +NGINX_INSTALL_SOURCE="install_from_repo" + CLUSTER="" DOMAIN="" @@ -206,8 +208,8 @@ VERSION="latest" # Other formula versions we depend on POSTGRES_TAG="v0.44.0" -NGINX_TAG="v2.8.0" -DOCKER_TAG="v2.4.0" +NGINX_TAG="v2.8.1" +DOCKER_TAG="v2.4.2" LOCALE_TAG="v0.3.4" LETSENCRYPT_TAG="v2.1.0" @@ -261,7 +263,7 @@ if [ ! -z "${HOSTNAME_EXT}" ] ; then # Make sure that the value configured as IP_INT is a real IP on the system. # If we don't error out early here when there is a mismatch, the formula will # fail with hard to interpret nginx errors later on. - ip addr list |grep -q " ${IP_INT}/" + ip addr list |grep -q "${IP_INT}/" if [[ $? -ne 0 ]]; then echo "Unable to find the IP_INT address '${IP_INT}' on the system, please correct the value in local.params. Exiting..." exit 1 @@ -284,7 +286,7 @@ else case ${OS_ID} in "centos") echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019" - sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux + sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux setenforce permissive yum install -y curl git jq ;; @@ -474,6 +476,7 @@ if [ -d "${SOURCE_STATES_DIR}" ]; then s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g; s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g; s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g; + s#__SHELL_INT_IP__#${SHELL_INT_IP}#g; s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g; s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g; s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g; @@ -558,6 +561,10 @@ if [ -z "${ROLES}" ]; then echo " - nginx_workbench_configuration" >> ${P_DIR}/top.sls echo " - postgresql" >> ${P_DIR}/top.sls + # We need to tweak the Nginx's pillar depending whether we want plan nginx or nginx+passenger + NGINX_INSTALL_SOURCE="install_from_phusionpassenger" + sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls + if [ "${SSL_MODE}" = "lets-encrypt" ]; then if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls @@ -627,10 +634,13 @@ else # States # FIXME: https://dev.arvados.org/issues/17352 grep -q "postgres.client" ${S_DIR}/top.sls || echo " - postgres.client" >> ${S_DIR}/top.sls - grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls + if grep -q " - nginx.*$" ${S_DIR}/top.sls; then + sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${S_DIR}/top.sls + else + echo " - nginx.passenger" >> ${S_DIR}/top.sls + fi ### If we don't install and run LE before arvados-api-server, it fails and breaks everything ### after it. So we add this here as we are, after all, sharing the host for api and controller - # Currently, only available on config_examples/multi_host/aws if [ "${SSL_MODE}" = "lets-encrypt" ]; then if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls @@ -649,11 +659,23 @@ else grep -q "postgresql" ${P_DIR}/top.sls || echo " - postgresql" >> ${P_DIR}/top.sls grep -q "nginx_passenger" ${P_DIR}/top.sls || echo " - nginx_passenger" >> ${P_DIR}/top.sls grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo " - nginx_${R}_configuration" >> ${P_DIR}/top.sls + + # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger + NGINX_INSTALL_SOURCE="install_from_phusionpassenger" + sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls ;; "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy") # States - grep -q "nginx.passenger" ${S_DIR}/top.sls || echo " - nginx.passenger" >> ${S_DIR}/top.sls - # Currently, only available on config_examples/multi_host/aws + if [ "${R}" = "workbench" ]; then + NGINX_INSTALL_SOURCE="install_from_phusionpassenger" + if grep -q " - nginx$" ${S_DIR}/top.sls; then + sed -i s/"^ - nginx.*$"/" - nginx.passenger"/g ${S_DIR}/top.sls + else + echo " - nginx.passenger" >> ${S_DIR}/top.sls + fi + else + grep -q "nginx" ${S_DIR}/top.sls || echo " - nginx" >> ${S_DIR}/top.sls + fi if [ "${SSL_MODE}" = "lets-encrypt" ]; then if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then grep -q "aws_credentials" ${S_DIR}/top.sls || echo " - aws_credentials" >> ${S_DIR}/top.sls @@ -685,7 +707,6 @@ else grep -q "nginx_collections_configuration" ${P_DIR}/top.sls || echo " - nginx_collections_configuration" >> ${P_DIR}/top.sls fi - # Currently, only available on config_examples/multi_host/aws if [ "${SSL_MODE}" = "lets-encrypt" ]; then if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then grep -q "aws_credentials" ${P_DIR}/top.sls || echo " - aws_credentials" >> ${P_DIR}/top.sls @@ -727,6 +748,8 @@ else grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo " - ${R}" >> ${P_DIR}/extra_custom_certs.sls fi fi + # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger + sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls ;; "shell") # States