fix(provision): force user to properly set cluster & domain parameters

Also improved single host installation options (single, multiple hostnames)

refs #17246
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli@curii.com>

diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 666c6c48ff192b7d6b29efe90e14f5a82e79d64f..6a093b1526105a5a6d4d28cdeb64c6533eccf7b6 100644 (file)
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -11,10 +11,45 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
   config.ssh.insert_key = false
   config.ssh.forward_x11 = true
 
   config.ssh.insert_key = false
   config.ssh.forward_x11 = true
 

-  # A single_host multiple_hostnames example
-  config.vm.define "arvados-sh-mn" do |arv|
+##   # A single_host multiple_hostnames example
+##   config.vm.define "arvados-sh-mn" do |arv|
+##     arv.vm.box = "bento/debian-10"
+##     arv.vm.hostname = "harpo"
+##     # CPU/RAM
+##     config.vm.provider :virtualbox do |v|
+##       v.memory = 2048
+##       v.cpus = 2
+##     end
+##
+##     # Networking
+##     # WEBUI PORT
+##     arv.vm.network "forwarded_port", guest: 8443, host: 8443
+##     # KEEPPROXY
+##     arv.vm.network "forwarded_port", guest: 25101, host: 25101
+##     # KEEPWEB
+##     arv.vm.network "forwarded_port", guest: 9002, host: 9002
+##     # WEBSOCKET
+##     arv.vm.network "forwarded_port", guest: 8002, host: 8002
+##     arv.vm.provision "shell",
+##                      inline: "sed 's#cluster_fixme_or_this_wont_work#harpo#g;
+##                                    s#domain_fixme_or_this_wont_work#local#g;
+##                                    s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \
+##                                    /vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
+##                                    # s#production#development#g;
+##     arv.vm.provision "shell",
+##                      path: "provision.sh",
+##                      args: [
+##                        # "--debug",
+##                        "--config /tmp/local.params.single_host_multiple_hostnames",
+##                        "--test",
+##                        "--vagrant"
+##                      ].join(" ")
+##   end
+
+  # A single_host single_hostname example
+  config.vm.define "arvados-sh-sn" do |arv|

     arv.vm.box = "bento/debian-10"
     arv.vm.box = "bento/debian-10"

-    arv.vm.hostname = "harpo.local"
+    arv.vm.hostname = "zeppo"

     # CPU/RAM
     config.vm.provider :virtualbox do |v|
       v.memory = 2048
     # CPU/RAM
     config.vm.provider :virtualbox do |v|
       v.memory = 2048


@@ -22,66 +57,33 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     end
 
     # Networking
     end
 
     # Networking

-    # WEBUI PORT
-    arv.vm.network "forwarded_port", guest: 8443, host: 8443
-    # KEEPPROXY
-    arv.vm.network "forwarded_port", guest: 25101, host: 25101
-    # KEEPWEB
-    arv.vm.network "forwarded_port", guest: 9002, host: 9002
-    # WEBSOCKET
-    arv.vm.network "forwarded_port", guest: 8002, host: 8002
+    arv.vm.network "forwarded_port", guest: 9443, host: 9443
+    arv.vm.network "forwarded_port", guest: 9444, host: 9444
+    arv.vm.network "forwarded_port", guest: 9445, host: 9445
+    arv.vm.network "forwarded_port", guest: 35101, host: 35101
+    arv.vm.network "forwarded_port", guest: 10002, host: 10002
+    arv.vm.network "forwarded_port", guest: 14202, host: 14202
+    arv.vm.network "forwarded_port", guest: 18002, host: 18002

     arv.vm.provision "shell",
     arv.vm.provision "shell",

-                     inline: "sed 's#fixme#harpo#g;
-                                   s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \
-                                   /vagrant/local.params.example > /vagrant/local.params.single_host_multiple_hostnames"
+                     inline: "sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g;
+                                   s#cluster_fixme_or_this_wont_work#harpo#g;
+                                   s#domain_fixme_or_this_wont_work#local#g;
+                                   s#CONFIG_DIR=\"config_examples/single_host/multiple_hostnames\"#CONFIG_DIR=\"config_examples/single_host/single_hostname\"#g;
+                                   s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=9443#g;
+                                   s#KEEP_EXT_SSL_PORT=25101#KEEP_EXT_SSL_PORT=35101#g;
+                                   s#KEEPWEB_EXT_SSL_PORT=9002#KEEPWEB_EXT_SSL_PORT=11002#g;
+                                   s#WEBSHELL_EXT_SSL_PORT=4202#WEBSHELL_EXT_SSL_PORT=14202#g;
+                                   s#WEBSOCKET_EXT_SSL_PORT=8002#WEBSOCKET_EXT_SSL_PORT=18002#g;
+                                   s#WORKBENCH1_EXT_SSL_PORT=443#WORKBENCH1_EXT_SSL_PORT=9444#g;
+                                   s#WORKBENCH2_EXT_SSL_PORT=3001#WORKBENCH2_EXT_SSL_PORT=9445#g;' \
+                                   /vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"

     arv.vm.provision "shell",
                      path: "provision.sh",
                      args: [
                        # "--debug",
     arv.vm.provision "shell",
                      path: "provision.sh",
                      args: [
                        # "--debug",

-                       "--config /vagrant/local.params.single_host_multiple_hostnames",
+                       "--config /tmp/local.params.single_host_single_hostname",

                        "--test",
                        "--vagrant"
                      ].join(" ")
   end
                        "--test",
                        "--vagrant"
                      ].join(" ")
   end

-
-  ## # A single_host single_hostname example
-  ## config.vm.define "arvados-sh-sn" do |arv|
-  ##   arv.vm.box = "bento/debian-10"
-  ##   arv.vm.hostname = "zeppo.local"
-  ##   # CPU/RAM
-  ##   config.vm.provider :virtualbox do |v|
-  ##     v.memory = 2048
-  ##     v.cpus = 2
-  ##   end
-
-  ##   # Networking
-  ##   arv.vm.network "forwarded_port", guest: 9443, host: 9443
-  ##   arv.vm.network "forwarded_port", guest: 9444, host: 9444
-  ##   arv.vm.network "forwarded_port", guest: 9445, host: 9445
-  ##   arv.vm.network "forwarded_port", guest: 35101, host: 35101
-  ##   arv.vm.network "forwarded_port", guest: 10002, host: 10002
-  ##   arv.vm.network "forwarded_port", guest: 14202, host: 14202
-  ##   arv.vm.network "forwarded_port", guest: 18002, host: 18002
-  ##   arv.vm.provision "shell",
-  ##                    inline: "sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local.cluster\"#g;
-  ##                                  s#CLUSTER=\"fixme\"#CLUSTER=\"zeppo\"#g;
-  ##                                  s#DOMAIN=\"some.domain\"#DOMAIN=\"local.cluster\"#g;
-  ##                                  s#CONFIG_DIR=\"config_examples/single_host/multiple_hostnames\"#CONFIG_DIR=\"config_examples/single_host/single_hostname\"#g;
-  ##                                  s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=9443#g;
-  ##                                  s#KEEP_EXT_SSL_PORT=25101#KEEP_EXT_SSL_PORT=35101#g;
-  ##                                  s#KEEPWEB_EXT_SSL_PORT=9002#KEEPWEB_EXT_SSL_PORT=11002#g;
-  ##                                  s#WEBSHELL_EXT_SSL_PORT=4202#WEBSHELL_EXT_SSL_PORT=14202#g;
-  ##                                  s#WEBSOCKET_EXT_SSL_PORT=8002#WEBSOCKET_EXT_SSL_PORT=18002#g;
-  ##                                  s#WORKBENCH1_EXT_SSL_PORT=443#WORKBENCH1_EXT_SSL_PORT=9444#g;
-  ##                                  s#WORKBENCH2_EXT_SSL_PORT=3001#WORKBENCH2_EXT_SSL_PORT=9445#g;' \
-  ##                                 /vagrant/local.params.example > /vagrant/local.params.single_host_single_hostname"
-  ##   arv.vm.provision "shell",
-  ##                    path: "provision.sh",
-  ##                    args: [
-  ##                      # "--debug",
-  ##                      "--config /vagrant/local.params.single_host_single_hostname",
-  ##                      "--test",
-  ##                      "--vagrant"
-  ##                    ].join(" ")
-  ## end

 end
 end

diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
index 31d3a0d5044b2d445cefceae30897e0db229cbb5..8fcad011667fd3f1c1e3445dde06d637fa90ed09 100644 (file)
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
@@ -81,6 +81,7 @@ arvados:
       system_root: __SYSTEM_ROOT_TOKEN__
       management: __MANAGEMENT_TOKEN__
       anonymous_user: __ANONYMOUS_USER_TOKEN__
       system_root: __SYSTEM_ROOT_TOKEN__
       management: __MANAGEMENT_TOKEN__
       anonymous_user: __ANONYMOUS_USER_TOKEN__

+      rails_secret: YDLxHf4GqqmLXYAMgndrAmFEdqgC0sBqX7TEjMN2rw9D6EVwgx

 
     ### KEYS
     secrets:
 
     ### KEYS
     secrets:

diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls
index 7e3957c575eacd242c193c79a90be2d95ebddddb..eac85452358a6694aa1d9995ea47faaef66f11ed 100644 (file)
--- a/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls
@@ -29,4 +29,5 @@ arvados_test_salt_states_examples_single_host_etc_hosts_host_present:
         ]
       %}
       - {{ entry }}
         ]
       %}
       - {{ entry }}

+      - {{ entry }}.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}

       {%- endfor %}
       {%- endfor %}

diff --git a/tools/salt-install/local.params.example b/tools/salt-install/local.params.example.single_host_multiple_hostnames
similarity index 88%
rename from tools/salt-install/local.params.example
rename to tools/salt-install/local.params.example.single_host_multiple_hostnames
index 88d6a75d680bd3b860bdae203240f5924f2e1e3d..78c26af0efe827338b2a64a97b23864bec790651 100644 (file)
--- a/tools/salt-install/local.params.example
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -5,11 +5,11 @@
 
 # These are the basic parameters to configure the installation
 
 
 # These are the basic parameters to configure the installation
 

-# The 5 letters name you want to give your cluster
-CLUSTER="fixme"
+# The FIVE ALPHANUMERIC CHARACTERS name you want to give your cluster
+CLUSTER="cluster_fixme_or_this_wont_work"

 
 # The domainname you want tou give to your cluster's hosts
 
 # The domainname you want tou give to your cluster's hosts

-DOMAIN="some.domain"
+DOMAIN="domain_fixme_or_this_wont_work"

 
 # When setting the cluster in a single host, you can use a single hostname
 # to access all the instances. When using virtualization (ie AWS), this should be
 
 # When setting the cluster in a single host, you can use a single hostname
 # to access all the instances. When using virtualization (ie AWS), this should be


@@ -36,7 +36,7 @@ INITIAL_USER="admin"
 
 # If not specified, the initial user email will be composed as
 # INITIAL_USER@CLUSTER.DOMAIN
 
 # If not specified, the initial user email will be composed as
 # INITIAL_USER@CLUSTER.DOMAIN

-INITIAL_USER_EMAIL="admin@fixme.localdomain"
+INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work"

 INITIAL_USER_PASSWORD="password"
 
 # YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS
 INITIAL_USER_PASSWORD="password"
 
 # YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS


@@ -51,7 +51,8 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 # variable, the single_host, multiple_hostnames config will be used
 # CONFIG_DIR="config_examples/single_host/single_hostname"
 CONFIG_DIR="config_examples/single_host/multiple_hostnames"
 # variable, the single_host, multiple_hostnames config will be used
 # CONFIG_DIR="config_examples/single_host/single_hostname"
 CONFIG_DIR="config_examples/single_host/multiple_hostnames"

-# Extra states to pply. iIf you use your own subdir, change this value accordingly
+# Extra states to apply. If you use your own subdir, change this value accordingly
+# This is the value for the single_host/multiple_hostnames example

 EXTRA_STATES_DIR="${F_DIR}/arvados-formula/test/salt/states/examples/single_host"
 
 # When using the single_host/single_hostname example, change to this one
 EXTRA_STATES_DIR="${F_DIR}/arvados-formula/test/salt/states/examples/single_host"
 
 # When using the single_host/single_hostname example, change to this one

diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
new file mode 100644 (file)
index 0000000..110d794
--- /dev/null
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -0,0 +1,80 @@
+##########################################################
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+# These are the basic parameters to configure the installation
+
+# The FIVE ALPHANUMERIC CHARACTERS name you want to give your cluster
+CLUSTER="cluster_fixme_or_this_wont_work"
+
+# The domainname you want tou give to your cluster's hosts
+DOMAIN="domain_fixme_or_this_wont_work"
+
+# When setting the cluster in a single host, you can use a single hostname
+# to access all the instances. When using virtualization (ie AWS), this should be
+# the EXTERNAL/PUBLIC hostname for the instance.
+# If empty, the INTERNAL HOST IP will be used
+HOSTNAME_EXT=""
+# The internal hostname for the host. In the example files, only used in the
+# single_host/single_hostname example
+HOSTNAME_INT="127.0.1.1"
+# Host SSL port where you want to point your browser to access Arvados
+# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
+# You can point it to another port if desired
+# In Vagrant, make sure it matches what you set in the Vagrantfile (8443)
+CONTROLLER_EXT_SSL_PORT=9443
+KEEP_EXT_SSL_PORT=35101
+# Both for collections and downloads
+KEEPWEB_EXT_SSL_PORT=11002
+WEBSHELL_EXT_SSL_PORT=14202
+WEBSOCKET_EXT_SSL_PORT=18002
+WORKBENCH1_EXT_SSL_PORT=9444
+WORKBENCH2_EXT_SSL_PORT=9445
+
+INITIAL_USER="admin"
+
+# If not specified, the initial user email will be composed as
+# INITIAL_USER@CLUSTER.DOMAIN
+INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work"
+INITIAL_USER_PASSWORD="password"
+
+# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS
+BLOB_SIGNING_KEY=blobsigningkeymushaveatleast32characters
+MANAGEMENT_TOKEN=managementtokenmushaveatleast32characters
+SYSTEM_ROOT_TOKEN=systemroottokenmushaveatleast32characters
+ANONYMOUS_USER_TOKEN=anonymoususertokenmushaveatleast32characters
+WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
+
+# The directory to check for the config files (pillars, states) you want to use.
+# There are a few examples under 'config_examples'. If you don't change this
+# variable, the single_host, multiple_hostnames config will be used
+# CONFIG_DIR="config_examples/single_host/single_hostname"
+CONFIG_DIR="config_examples/single_host/single_hostname"
+# Extra states to apply. If you use your own subdir, change this value accordingly
+# This is the value for the single_host/multiple_hostnames example
+# EXTRA_STATES_DIR="${F_DIR}/arvados-formula/test/salt/states/examples/single_host"
+
+# When using the single_host/single_hostname example, change to this one
+EXTRA_STATES_DIR="${CONFIG_DIR}/states"
+
+# Which release of Arvados repo you want to use
+RELEASE="production"
+# Which version of Arvados you want to install. Defaults to 'latest'
+# in the desired repo
+VERSION="latest"
+
+# This is an arvados-formula setting.
+# If branch is set, the script will switch to it before running salt
+# Usually not needed, only used for testing
+# BRANCH="master"
+
+##########################################################
+# Usually there's no need to modify things below this line
+
+# Formulas versions
+ARVADOS_TAG="v1.1.4"
+POSTGRES_TAG="v0.41.3"
+NGINX_TAG="v2.4.0"
+DOCKER_TAG="v1.0.0"
+LOCALE_TAG="v0.3.4"

diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 9b19854d039362bdcdbad39f45a06b4d2c78dfcc..5174f2398cbea0259e1329a40490f9166382cec9 100755 (executable)
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -151,6 +151,12 @@ else
   exit 1
 fi
 
   exit 1
 fi
 

+if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
+  echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified."
+  echo >&2 "Please, fix them and re-run the provision script."
+  exit 1
+fi
+

 if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
 if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"


@@ -192,9 +198,10 @@ git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-
 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 
 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 

+# If we want to try a specific branch of the formula

 if [ "x${BRANCH}" != "x" ]; then
   cd ${F_DIR}/arvados-formula || exit 1
 if [ "x${BRANCH}" != "x" ]; then
   cd ${F_DIR}/arvados-formula || exit 1

-  git checkout -t origin/"${BRANCH}"
+  git checkout -t origin/"${BRANCH}" -b "${BRANCH}"

   cd -
 fi
 
   cd -
 fi
 

diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index 6bc8422f8d1983c58db2757b2a68ed71c64e2957..53c51a2c5a097d2e8b45446ea26a7e2a26800f2d 100755 (executable)
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -10,7 +10,7 @@ export ARVADOS_API_HOST_INSECURE=true
 set -o pipefail
 
 # First, validate that the CA is installed and that we can query it with no errors.
 set -o pipefail
 
 # First, validate that the CA is installed and that we can query it with no errors.

-if ! curl -s -o /dev/null https://workbench.${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then
+if ! curl -s -o /dev/null https://${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then

   echo "The Arvados CA was not correctly installed. Although some components will work,"
   echo "others won't. Please verify that the CA cert file was installed correctly and"
   echo "retry running these tests."
   echo "The Arvados CA was not correctly installed. Although some components will work,"
   echo "others won't. Please verify that the CA cert file was installed correctly and"
   echo "retry running these tests."