Merge branch '18681-install-passenger-only-when-required'
authorJavier Bértoli <jbertoli@curii.com>
Thu, 17 Mar 2022 18:19:06 +0000 (15:19 -0300)
committerJavier Bértoli <jbertoli@curii.com>
Thu, 17 Mar 2022 18:19:06 +0000 (15:19 -0300)
closes #18681
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli@curii.com>

tools/salt-install/Vagrantfile
tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
tools/salt-install/local.params.example.multiple_hosts
tools/salt-install/local.params.example.single_host_multiple_hostnames
tools/salt-install/local.params.example.single_host_single_hostname
tools/salt-install/provision.sh

index a8dd34534303cb4266ed51c0bd19eae5f9d1c61f..1573b6862b3d50f849a54ae4f6a9ab9858754281 100644 (file)
@@ -41,6 +41,9 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                                     /vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
                                     # s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;' \
 
+     arv.vm.provision "shell",
+                      inline: "cp -vr /tmp/local.params.single_host_multiple_hostnames /tmp/local.params.single_host_multiple_hostnames.falla;
+                               cp -vr /vagrant/centos7-local.params.single_host_single_hostname-f258b604f831bb3bd7fab506c670b975ae8e4118 /tmp/local.params.single_host_multiple_hostnames"
      arv.vm.provision "shell",
                       path: "provision.sh",
                       args: [
@@ -54,7 +57,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
    # A single_host single_hostname example
    config.vm.define "arvados-sh-sn" do |arv|
-     arv.vm.box = "bento/debian-11"
+     #arv.vm.box = "bento/centos-7"
+     arv.vm.box = "bento/ubuntu-20.04"
      arv.vm.hostname = "zeppo"
      # CPU/RAM
      config.vm.provider :virtualbox do |v|
@@ -80,12 +84,15 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
      arv.vm.provision "shell",
                       inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
                                cp -vr /vagrant/tests /home/vagrant/tests;
-                               sed 's#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"zeppo.local\"#g;
-                                   's#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g;
-                                    s#cluster_fixme_or_this_wont_work#zeppo#g;
-                                    s#domain_fixme_or_this_wont_work#local#g;' \
+                               sed 's#cluster_fixme_or_this_wont_work#cnts7#g;
+                                    s#domain_fixme_or_this_wont_work#local#g;
+                                    s#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"cnts7.local\"#g;
+                                    s#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g;
+                                    s#RELEASE=\"production\"#RELEASE=\"development\"#g;
+                                    s/# BRANCH=\"main\"/BRANCH=\"main\"/g;
+                                    s/# VERSION=.*$/VERSION=\"latest\"/g' \
                                     /vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
-                                    # s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
+
      arv.vm.provision "shell",
                       path: "provision.sh",
                       args: [
index d9644e5645833257eb329ba452f879ec9daf67d3..fbd42bd7a36fba39680d629f46ad91b2da04bc1e 100644 (file)
@@ -15,7 +15,7 @@
 
 ### NGINX
 nginx:
-  install_from_phusionpassenger: true
+  __NGINX_INSTALL_SOURCE__: true
   lookup:
     passenger_package: {{ passenger_pkg }}
   ### PASSENGER
@@ -25,11 +25,15 @@ nginx:
   ### SERVER
   server:
     config:
+      # As we now differentiate where passenger is required or not, we need to
+      # load this module conditionally, so we add this conditional just to use
+      # the same pillar file
+      {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
       # This is required to get the passenger module loaded
       # In Debian it can be done with this
       # include: 'modules-enabled/*.conf'
       load_module: {{ passenger_mod }}
-
+      {% endif %}
       worker_processes: 4
 
   ### SNIPPETS
index a0da9a1c057777e009eb90bc3732c2c46023bf00..e06ddd041c9acb4d01a1bab8a3deb8de6253f287 100644 (file)
@@ -6,7 +6,7 @@
 ### POSTGRESQL
 postgres:
   use_upstream_repo: true
-  version: '11'
+  version: '12'
   postgresconf: |-
     listen_addresses = '*'  # listen on all interfaces
   acls:
index 886aab87648b430eacd6bc6d59305a6389ff67a6..dbf21c265129d9721fa50e63b17bb5a3bfd612b6 100644 (file)
@@ -15,7 +15,7 @@
 
 ### NGINX
 nginx:
-  install_from_phusionpassenger: true
+  __NGINX_INSTALL_SOURCE__: true
   lookup:
     passenger_package: {{ passenger_pkg }}
   ### PASSENGER
@@ -25,11 +25,15 @@ nginx:
   ### SERVER
   server:
     config:
+      # As we now differentiate where passenger is required or not, we need to
+      # load this module conditionally, so we add this conditional just to use
+      # the same pillar file
+      {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
       # This is required to get the passenger module loaded
       # In Debian it can be done with this
       # include: 'modules-enabled/*.conf'
       load_module: {{ passenger_mod }}
-
+      {% endif %}
       worker_processes: 4
 
   ### SNIPPETS
index bce51f29e0b33a43316d70c28c04cd544e6f028e..c25720c60a5fb17bc7aa3d5199a6ba79b82066db 100644 (file)
@@ -15,7 +15,7 @@
 
 ### NGINX
 nginx:
-  install_from_phusionpassenger: true
+  __NGINX_INSTALL_SOURCE__: true
   lookup:
     passenger_package: {{ passenger_pkg }}
   ### PASSENGER
@@ -25,11 +25,15 @@ nginx:
   ### SERVER
   server:
     config:
+      # As we now differentiate where passenger is required or not, we need to
+      # load this module conditionally, so we add this conditional just to use
+      # the same pillar file
+      {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
       # This is required to get the passenger module loaded
       # In Debian it can be done with this
       # include: 'modules-enabled/*.conf'
       load_module: {{ passenger_mod }}
-
+      {% endif %}
       worker_processes: 4
 
   ### SNIPPETS
index 221e7b35ebc7952b4b680e7db2691ddabf7843dc..32d1f8bb961d7205a3b03512051bad8d2a431152 100644 (file)
@@ -119,8 +119,8 @@ RELEASE="production"
 
 # Formulas versions
 # ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="v2.8.0"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
index 77015248c1e7cffae398816de43724385bcde242..d6bfb102e938eaa4150fd0d2af8e2f420d1597b1 100644 (file)
@@ -70,8 +70,8 @@ RELEASE="production"
 
 # Formulas versions
 # ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="v2.8.0"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
index a1cd491bc21bac9536a57dbb022851861f68da12..b6c7e5f7a5d9c4fdd310e32e6dabfe3df5eb9379 100644 (file)
@@ -79,7 +79,7 @@ RELEASE="production"
 # Formulas versions
 # ARVADOS_TAG="2.2.0"
 # POSTGRES_TAG="v0.44.0"
-# NGINX_TAG="v2.8.0"
-# DOCKER_TAG="v2.4.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
index 2d33cac57bda6cac06e8b644ea6eb9a5362603ee..0f3c9a14117b964907259d628afbb6de32f1e58d 100755 (executable)
@@ -165,6 +165,8 @@ LOG_LEVEL="info"
 CONTROLLER_EXT_SSL_PORT=443
 TESTS_DIR="tests"
 
+NGINX_INSTALL_SOURCE="install_from_repo"
+
 CLUSTER=""
 DOMAIN=""
 
@@ -206,8 +208,8 @@ VERSION="latest"
 
 # Other formula versions we depend on
 POSTGRES_TAG="v0.44.0"
-NGINX_TAG="v2.8.0"
-DOCKER_TAG="v2.4.0"
+NGINX_TAG="v2.8.1"
+DOCKER_TAG="v2.4.2"
 LOCALE_TAG="v0.3.4"
 LETSENCRYPT_TAG="v2.1.0"
 
@@ -261,7 +263,7 @@ if [ ! -z "${HOSTNAME_EXT}" ] ; then
   # Make sure that the value configured as IP_INT is a real IP on the system.
   # If we don't error out early here when there is a mismatch, the formula will
   # fail with hard to interpret nginx errors later on.
-  ip addr list |grep -q " ${IP_INT}/"
+  ip addr list |grep -q "${IP_INT}/"
   if [[ $? -ne 0 ]]; then
     echo "Unable to find the IP_INT address '${IP_INT}' on the system, please correct the value in local.params. Exiting..."
     exit 1
@@ -284,7 +286,7 @@ else
   case ${OS_ID} in
     "centos")
       echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
-      sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+      sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
       setenforce permissive
       yum install -y  curl git jq
       ;;
@@ -474,6 +476,7 @@ if [ -d "${SOURCE_STATES_DIR}" ]; then
          s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
          s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
          s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
+         s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
          s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
          s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
          s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
@@ -558,6 +561,10 @@ if [ -z "${ROLES}" ]; then
   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
   echo "    - postgresql" >> ${P_DIR}/top.sls
 
+  # We need to tweak the Nginx's pillar depending whether we want plan nginx or nginx+passenger
+  NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+  sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
+
   if [ "${SSL_MODE}" = "lets-encrypt" ]; then
     if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
       grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
@@ -627,10 +634,13 @@ else
         # States
         # FIXME: https://dev.arvados.org/issues/17352
         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
-        grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
+        if grep -q "    - nginx.*$" ${S_DIR}/top.sls; then
+          sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${S_DIR}/top.sls
+        else
+          echo "    - nginx.passenger" >> ${S_DIR}/top.sls
+        fi
         ### If we don't install and run LE before arvados-api-server, it fails and breaks everything
         ### after it. So we add this here as we are, after all, sharing the host for api and controller
-        # Currently, only available on config_examples/multi_host/aws
         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
@@ -649,11 +659,23 @@ else
         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
+
+        # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
+        NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+        sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
       ;;
       "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
         # States
-        grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
-        # Currently, only available on config_examples/multi_host/aws
+        if [ "${R}" = "workbench" ]; then
+          NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+          if grep -q "    - nginx$" ${S_DIR}/top.sls; then
+            sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${S_DIR}/top.sls
+          else
+            echo "    - nginx.passenger" >> ${S_DIR}/top.sls
+          fi
+        else
+          grep -q "nginx" ${S_DIR}/top.sls || echo "    - nginx" >> ${S_DIR}/top.sls
+        fi
         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
@@ -685,7 +707,6 @@ else
           grep -q "nginx_collections_configuration" ${P_DIR}/top.sls || echo "    - nginx_collections_configuration" >> ${P_DIR}/top.sls
         fi
 
-        # Currently, only available on config_examples/multi_host/aws
         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
             grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
@@ -727,6 +748,8 @@ else
             grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo "  - ${R}" >> ${P_DIR}/extra_custom_certs.sls
           fi
         fi
+        # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
+        sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
       ;;
       "shell")
         # States