From: Javier Bértoli Date: Tue, 16 Feb 2021 14:21:20 +0000 (-0300) Subject: fix(provision): force user to properly set cluster & domain parameters X-Git-Tag: 2.2.0~65^2~21 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/1a0cdc10f409fe410594f62a252c1aa5f264f345 fix(provision): force user to properly set cluster & domain parameters Also improved single host installation options (single, multiple hostnames) refs #17246 Arvados-DCO-1.1-Signed-off-by: Javier Bértoli --- diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile index 666c6c48ff..6a093b1526 100644 --- a/tools/salt-install/Vagrantfile +++ b/tools/salt-install/Vagrantfile @@ -11,10 +11,45 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| config.ssh.insert_key = false config.ssh.forward_x11 = true - # A single_host multiple_hostnames example - config.vm.define "arvados-sh-mn" do |arv| +## # A single_host multiple_hostnames example +## config.vm.define "arvados-sh-mn" do |arv| +## arv.vm.box = "bento/debian-10" +## arv.vm.hostname = "harpo" +## # CPU/RAM +## config.vm.provider :virtualbox do |v| +## v.memory = 2048 +## v.cpus = 2 +## end +## +## # Networking +## # WEBUI PORT +## arv.vm.network "forwarded_port", guest: 8443, host: 8443 +## # KEEPPROXY +## arv.vm.network "forwarded_port", guest: 25101, host: 25101 +## # KEEPWEB +## arv.vm.network "forwarded_port", guest: 9002, host: 9002 +## # WEBSOCKET +## arv.vm.network "forwarded_port", guest: 8002, host: 8002 +## arv.vm.provision "shell", +## inline: "sed 's#cluster_fixme_or_this_wont_work#harpo#g; +## s#domain_fixme_or_this_wont_work#local#g; +## s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \ +## /vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames" +## # s#production#development#g; +## arv.vm.provision "shell", +## path: "provision.sh", +## args: [ +## # "--debug", +## "--config /tmp/local.params.single_host_multiple_hostnames", +## "--test", +## "--vagrant" +## ].join(" ") +## end + + # A single_host single_hostname example + config.vm.define "arvados-sh-sn" do |arv| arv.vm.box = "bento/debian-10" - arv.vm.hostname = "harpo.local" + arv.vm.hostname = "zeppo" # CPU/RAM config.vm.provider :virtualbox do |v| v.memory = 2048 @@ -22,66 +57,33 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| end # Networking - # WEBUI PORT - arv.vm.network "forwarded_port", guest: 8443, host: 8443 - # KEEPPROXY - arv.vm.network "forwarded_port", guest: 25101, host: 25101 - # KEEPWEB - arv.vm.network "forwarded_port", guest: 9002, host: 9002 - # WEBSOCKET - arv.vm.network "forwarded_port", guest: 8002, host: 8002 + arv.vm.network "forwarded_port", guest: 9443, host: 9443 + arv.vm.network "forwarded_port", guest: 9444, host: 9444 + arv.vm.network "forwarded_port", guest: 9445, host: 9445 + arv.vm.network "forwarded_port", guest: 35101, host: 35101 + arv.vm.network "forwarded_port", guest: 10002, host: 10002 + arv.vm.network "forwarded_port", guest: 14202, host: 14202 + arv.vm.network "forwarded_port", guest: 18002, host: 18002 arv.vm.provision "shell", - inline: "sed 's#fixme#harpo#g; - s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g' \ - /vagrant/local.params.example > /vagrant/local.params.single_host_multiple_hostnames" + inline: "sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g; + s#cluster_fixme_or_this_wont_work#harpo#g; + s#domain_fixme_or_this_wont_work#local#g; + s#CONFIG_DIR=\"config_examples/single_host/multiple_hostnames\"#CONFIG_DIR=\"config_examples/single_host/single_hostname\"#g; + s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=9443#g; + s#KEEP_EXT_SSL_PORT=25101#KEEP_EXT_SSL_PORT=35101#g; + s#KEEPWEB_EXT_SSL_PORT=9002#KEEPWEB_EXT_SSL_PORT=11002#g; + s#WEBSHELL_EXT_SSL_PORT=4202#WEBSHELL_EXT_SSL_PORT=14202#g; + s#WEBSOCKET_EXT_SSL_PORT=8002#WEBSOCKET_EXT_SSL_PORT=18002#g; + s#WORKBENCH1_EXT_SSL_PORT=443#WORKBENCH1_EXT_SSL_PORT=9444#g; + s#WORKBENCH2_EXT_SSL_PORT=3001#WORKBENCH2_EXT_SSL_PORT=9445#g;' \ + /vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname" arv.vm.provision "shell", path: "provision.sh", args: [ # "--debug", - "--config /vagrant/local.params.single_host_multiple_hostnames", + "--config /tmp/local.params.single_host_single_hostname", "--test", "--vagrant" ].join(" ") end - - ## # A single_host single_hostname example - ## config.vm.define "arvados-sh-sn" do |arv| - ## arv.vm.box = "bento/debian-10" - ## arv.vm.hostname = "zeppo.local" - ## # CPU/RAM - ## config.vm.provider :virtualbox do |v| - ## v.memory = 2048 - ## v.cpus = 2 - ## end - - ## # Networking - ## arv.vm.network "forwarded_port", guest: 9443, host: 9443 - ## arv.vm.network "forwarded_port", guest: 9444, host: 9444 - ## arv.vm.network "forwarded_port", guest: 9445, host: 9445 - ## arv.vm.network "forwarded_port", guest: 35101, host: 35101 - ## arv.vm.network "forwarded_port", guest: 10002, host: 10002 - ## arv.vm.network "forwarded_port", guest: 14202, host: 14202 - ## arv.vm.network "forwarded_port", guest: 18002, host: 18002 - ## arv.vm.provision "shell", - ## inline: "sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local.cluster\"#g; - ## s#CLUSTER=\"fixme\"#CLUSTER=\"zeppo\"#g; - ## s#DOMAIN=\"some.domain\"#DOMAIN=\"local.cluster\"#g; - ## s#CONFIG_DIR=\"config_examples/single_host/multiple_hostnames\"#CONFIG_DIR=\"config_examples/single_host/single_hostname\"#g; - ## s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=9443#g; - ## s#KEEP_EXT_SSL_PORT=25101#KEEP_EXT_SSL_PORT=35101#g; - ## s#KEEPWEB_EXT_SSL_PORT=9002#KEEPWEB_EXT_SSL_PORT=11002#g; - ## s#WEBSHELL_EXT_SSL_PORT=4202#WEBSHELL_EXT_SSL_PORT=14202#g; - ## s#WEBSOCKET_EXT_SSL_PORT=8002#WEBSOCKET_EXT_SSL_PORT=18002#g; - ## s#WORKBENCH1_EXT_SSL_PORT=443#WORKBENCH1_EXT_SSL_PORT=9444#g; - ## s#WORKBENCH2_EXT_SSL_PORT=3001#WORKBENCH2_EXT_SSL_PORT=9445#g;' \ - ## /vagrant/local.params.example > /vagrant/local.params.single_host_single_hostname" - ## arv.vm.provision "shell", - ## path: "provision.sh", - ## args: [ - ## # "--debug", - ## "--config /vagrant/local.params.single_host_single_hostname", - ## "--test", - ## "--vagrant" - ## ].join(" ") - ## end end diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls index 31d3a0d504..8fcad01166 100644 --- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls +++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls @@ -81,6 +81,7 @@ arvados: system_root: __SYSTEM_ROOT_TOKEN__ management: __MANAGEMENT_TOKEN__ anonymous_user: __ANONYMOUS_USER_TOKEN__ + rails_secret: YDLxHf4GqqmLXYAMgndrAmFEdqgC0sBqX7TEjMN2rw9D6EVwgx ### KEYS secrets: diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls index 7e3957c575..eac8545235 100644 --- a/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls +++ b/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls @@ -29,4 +29,5 @@ arvados_test_salt_states_examples_single_host_etc_hosts_host_present: ] %} - {{ entry }} + - {{ entry }}.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} {%- endfor %} diff --git a/tools/salt-install/local.params.example b/tools/salt-install/local.params.example.single_host_multiple_hostnames similarity index 88% rename from tools/salt-install/local.params.example rename to tools/salt-install/local.params.example.single_host_multiple_hostnames index 88d6a75d68..78c26af0ef 100644 --- a/tools/salt-install/local.params.example +++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames @@ -5,11 +5,11 @@ # These are the basic parameters to configure the installation -# The 5 letters name you want to give your cluster -CLUSTER="fixme" +# The FIVE ALPHANUMERIC CHARACTERS name you want to give your cluster +CLUSTER="cluster_fixme_or_this_wont_work" # The domainname you want tou give to your cluster's hosts -DOMAIN="some.domain" +DOMAIN="domain_fixme_or_this_wont_work" # When setting the cluster in a single host, you can use a single hostname # to access all the instances. When using virtualization (ie AWS), this should be @@ -36,7 +36,7 @@ INITIAL_USER="admin" # If not specified, the initial user email will be composed as # INITIAL_USER@CLUSTER.DOMAIN -INITIAL_USER_EMAIL="admin@fixme.localdomain" +INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work" INITIAL_USER_PASSWORD="password" # YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS @@ -51,7 +51,8 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters # variable, the single_host, multiple_hostnames config will be used # CONFIG_DIR="config_examples/single_host/single_hostname" CONFIG_DIR="config_examples/single_host/multiple_hostnames" -# Extra states to pply. iIf you use your own subdir, change this value accordingly +# Extra states to apply. If you use your own subdir, change this value accordingly +# This is the value for the single_host/multiple_hostnames example EXTRA_STATES_DIR="${F_DIR}/arvados-formula/test/salt/states/examples/single_host" # When using the single_host/single_hostname example, change to this one diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname new file mode 100644 index 0000000000..110d794296 --- /dev/null +++ b/tools/salt-install/local.params.example.single_host_single_hostname @@ -0,0 +1,80 @@ +########################################################## +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: CC-BY-SA-3.0 + +# These are the basic parameters to configure the installation + +# The FIVE ALPHANUMERIC CHARACTERS name you want to give your cluster +CLUSTER="cluster_fixme_or_this_wont_work" + +# The domainname you want tou give to your cluster's hosts +DOMAIN="domain_fixme_or_this_wont_work" + +# When setting the cluster in a single host, you can use a single hostname +# to access all the instances. When using virtualization (ie AWS), this should be +# the EXTERNAL/PUBLIC hostname for the instance. +# If empty, the INTERNAL HOST IP will be used +HOSTNAME_EXT="" +# The internal hostname for the host. In the example files, only used in the +# single_host/single_hostname example +HOSTNAME_INT="127.0.1.1" +# Host SSL port where you want to point your browser to access Arvados +# Defaults to 443 for regular runs, and to 8443 when called in Vagrant. +# You can point it to another port if desired +# In Vagrant, make sure it matches what you set in the Vagrantfile (8443) +CONTROLLER_EXT_SSL_PORT=9443 +KEEP_EXT_SSL_PORT=35101 +# Both for collections and downloads +KEEPWEB_EXT_SSL_PORT=11002 +WEBSHELL_EXT_SSL_PORT=14202 +WEBSOCKET_EXT_SSL_PORT=18002 +WORKBENCH1_EXT_SSL_PORT=9444 +WORKBENCH2_EXT_SSL_PORT=9445 + +INITIAL_USER="admin" + +# If not specified, the initial user email will be composed as +# INITIAL_USER@CLUSTER.DOMAIN +INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work" +INITIAL_USER_PASSWORD="password" + +# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS +BLOB_SIGNING_KEY=blobsigningkeymushaveatleast32characters +MANAGEMENT_TOKEN=managementtokenmushaveatleast32characters +SYSTEM_ROOT_TOKEN=systemroottokenmushaveatleast32characters +ANONYMOUS_USER_TOKEN=anonymoususertokenmushaveatleast32characters +WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters + +# The directory to check for the config files (pillars, states) you want to use. +# There are a few examples under 'config_examples'. If you don't change this +# variable, the single_host, multiple_hostnames config will be used +# CONFIG_DIR="config_examples/single_host/single_hostname" +CONFIG_DIR="config_examples/single_host/single_hostname" +# Extra states to apply. If you use your own subdir, change this value accordingly +# This is the value for the single_host/multiple_hostnames example +# EXTRA_STATES_DIR="${F_DIR}/arvados-formula/test/salt/states/examples/single_host" + +# When using the single_host/single_hostname example, change to this one +EXTRA_STATES_DIR="${CONFIG_DIR}/states" + +# Which release of Arvados repo you want to use +RELEASE="production" +# Which version of Arvados you want to install. Defaults to 'latest' +# in the desired repo +VERSION="latest" + +# This is an arvados-formula setting. +# If branch is set, the script will switch to it before running salt +# Usually not needed, only used for testing +# BRANCH="master" + +########################################################## +# Usually there's no need to modify things below this line + +# Formulas versions +ARVADOS_TAG="v1.1.4" +POSTGRES_TAG="v0.41.3" +NGINX_TAG="v2.4.0" +DOCKER_TAG="v1.0.0" +LOCALE_TAG="v0.3.4" diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh index 9b19854d03..5174f2398c 100755 --- a/tools/salt-install/provision.sh +++ b/tools/salt-install/provision.sh @@ -151,6 +151,12 @@ else exit 1 fi +if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then + echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified." + echo >&2 "Please, fix them and re-run the provision script." + exit 1 +fi + if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then echo >&2 "ERROR: must be exactly 5 alphanumeric characters long" echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script" @@ -192,9 +198,10 @@ git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale- git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git +# If we want to try a specific branch of the formula if [ "x${BRANCH}" != "x" ]; then cd ${F_DIR}/arvados-formula || exit 1 - git checkout -t origin/"${BRANCH}" + git checkout -t origin/"${BRANCH}" -b "${BRANCH}" cd - fi diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh index 6bc8422f8d..53c51a2c5a 100755 --- a/tools/salt-install/tests/run-test.sh +++ b/tools/salt-install/tests/run-test.sh @@ -10,7 +10,7 @@ export ARVADOS_API_HOST_INSECURE=true set -o pipefail # First, validate that the CA is installed and that we can query it with no errors. -if ! curl -s -o /dev/null https://workbench.${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then +if ! curl -s -o /dev/null https://${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then echo "The Arvados CA was not correctly installed. Although some components will work," echo "others won't. Please verify that the CA cert file was installed correctly and" echo "retry running these tests."