tools/salt-install/provision.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14 set -x
  15
  16 # capture the directory that the script is running from
  17 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  18
  19 usage() {
  20   echo >&2
  21   echo >&2 "Usage: ${0} [-h] [-h]"
  22   echo >&2
  23   echo >&2 "${0} options:"
  24   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  25   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  26   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  27   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  28   echo >&2 "                                              Possible values are:"
  29   echo >&2 "                                                api"
  30   echo >&2 "                                                controller"
  31   echo >&2 "                                                dispatcher"
  32   echo >&2 "                                                keepproxy"
  33   echo >&2 "                                                keepstore"
  34   echo >&2 "                                                keepweb"
  35   echo >&2 "                                                shell"
  36   echo >&2 "                                                webshell"
  37   echo >&2 "                                                websocket"
  38   echo >&2 "                                                workbench"
  39   echo >&2 "                                                workbench2"
  40   echo >&2 "                                              Defaults to applying them all"
  41   echo >&2 "  -h, --help                                  Display this help and exit"
  42   echo >&2 "  --dump-config <dest_dir>                    Dumps the pillars and states to a directory"
  43   echo >&2 "                                              This parameter does not perform any installation at all. It's"
  44   echo >&2 "                                              intended to give you a parsed sot of configuration files so"
  45   echo >&2 "                                              you can inspect them or use them in you Saltstack infrastructure."
  46   echo >&2 "                                              It"
  47   echo >&2 "                                                - parses the pillar and states templates,"
  48   echo >&2 "                                                - downloads the helper formulas with their desired versions,"
  49   echo >&2 "                                                - prepares the 'top.sls' files both for pillars and states"
  50   echo >&2 "                                                  for the selected role/s"
  51   echo >&2 "                                                - writes the resulting files into <dest_dir>"
  52   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  53   echo >&2
  54 }
  55
  56 arguments() {
  57   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  58   if ! which getopt > /dev/null; then
  59     echo >&2 "GNU getopt is required to run this script. Please install it and re-reun it"
  60     exit 1
  61   fi
  62
  63   TEMP=$(getopt -o c:dhp:r:tv \
  64     --long config:,debug,dump-config:,help,roles:,test,vagrant \
  65     -n "${0}" -- "${@}")
  66
  67   if [ ${?} != 0 ];
  68     then echo "Please check the parameters you entered and re-run again"
  69     exit 1
  70   fi
  71   # Note the quotes around `$TEMP': they are essential!
  72   eval set -- "$TEMP"
  73
  74   while [ ${#} -ge 1 ]; do
  75     case ${1} in
  76       -c | --config)
  77         CONFIG_FILE=${2}
  78         shift 2
  79         ;;
  80       -d | --debug)
  81         LOG_LEVEL="debug"
  82         shift
  83         set -x
  84         ;;
  85       --dump-config)
  86         if [[ ${2} = /* ]]; then
  87           DUMP_SALT_CONFIG_DIR=${2}
  88         else
  89           DUMP_SALT_CONFIG_DIR=${PWD}/${2}
  90         fi
  91         ## states
  92         S_DIR="${DUMP_SALT_CONFIG_DIR}/salt"
  93         ## formulas
  94         F_DIR="${DUMP_SALT_CONFIG_DIR}/formulas"
  95         ## pillars
  96         P_DIR="${DUMP_SALT_CONFIG_DIR}/pillars"
  97         ## tests
  98         T_DIR="${DUMP_SALT_CONFIG_DIR}/tests"
  99         DUMP_CONFIG="yes"
 100         shift 2
 101         ;;
 102       -r | --roles)
 103         for i in ${2//,/ }
 104           do
 105             # Verify the role exists
 106             if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
 107               echo "The role '${i}' is not a valid role"
 108               usage
 109               exit 1
 110             fi
 111             ROLES="${ROLES} ${i}"
 112           done
 113           shift 2
 114         ;;
 115       -t | --test)
 116         TEST="yes"
 117         shift
 118         ;;
 119       -v | --vagrant)
 120         VAGRANT="yes"
 121         shift
 122         ;;
 123       --)
 124         shift
 125         break
 126         ;;
 127       *)
 128         usage
 129         exit 1
 130         ;;
 131     esac
 132   done
 133 }
 134
 135 CONFIG_FILE="${SCRIPT_DIR}/local.params"
 136 CONFIG_DIR="local_config_dir"
 137 DUMP_CONFIG="no"
 138 LOG_LEVEL="info"
 139 CONTROLLER_EXT_SSL_PORT=443
 140 TESTS_DIR="tests"
 141
 142 CLUSTER=""
 143 DOMAIN=""
 144
 145 # Hostnames/IPs used for single-host deploys
 146 HOSTNAME_EXT=""
 147 HOSTNAME_INT="127.0.1.1"
 148
 149 # Initial user setup
 150 INITIAL_USER=""
 151 INITIAL_USER_EMAIL=""
 152 INITIAL_USER_PASSWORD=""
 153
 154 CONTROLLER_EXT_SSL_PORT=8000
 155 KEEP_EXT_SSL_PORT=25101
 156 # Both for collections and downloads
 157 KEEPWEB_EXT_SSL_PORT=9002
 158 WEBSHELL_EXT_SSL_PORT=4202
 159 WEBSOCKET_EXT_SSL_PORT=8002
 160 WORKBENCH1_EXT_SSL_PORT=443
 161 WORKBENCH2_EXT_SSL_PORT=3001
 162
 163 ## These are ARVADOS-related parameters
 164 # For a stable release, change RELEASE "production" and VERSION to the
 165 # package version (including the iteration, e.g. X.Y.Z-1) of the
 166 # release.
 167 # The "local.params.example.*" files already set "RELEASE=production"
 168 # to deploy  production-ready packages
 169 RELEASE="development"
 170 VERSION="latest"
 171
 172 # These are arvados-formula-related parameters
 173 # An arvados-formula tag. For a stable release, this should be a
 174 # branch name (e.g. X.Y-dev) or tag for the release.
 175 # ARVADOS_TAG="2.2.0"
 176 # BRANCH="main"
 177
 178 # Other formula versions we depend on
 179 POSTGRES_TAG="v0.41.6"
 180 NGINX_TAG="temp-fix-missing-statements-in-pillar"
 181 DOCKER_TAG="v2.0.7"
 182 LOCALE_TAG="v0.3.4"
 183 LETSENCRYPT_TAG="v2.1.0"
 184
 185 # Salt's dir
 186 DUMP_SALT_CONFIG_DIR=""
 187 ## states
 188 S_DIR="/srv/salt"
 189 ## formulas
 190 F_DIR="/srv/formulas"
 191 ## pillars
 192 P_DIR="/srv/pillars"
 193 ## tests
 194 T_DIR="/tmp/cluster_tests"
 195
 196 arguments ${@}
 197
 198 if [ -s ${CONFIG_FILE} ]; then
 199   source ${CONFIG_FILE}
 200 else
 201   echo >&2 "You don't seem to have a config file with initial values."
 202   echo >&2 "Please create a '${CONFIG_FILE}' file as described in"
 203   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
 204   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
 205   exit 1
 206 fi
 207
 208 if [ ! -d ${CONFIG_DIR} ]; then
 209   echo >&2 "You don't seem to have a config directory with pillars and states."
 210   echo >&2 "Please create a '${CONFIG_DIR}' directory (as configured in your '${CONFIG_FILE}'). Please see"
 211   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
 212   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
 213   exit 1
 214 fi
 215
 216 if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
 217   echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified."
 218   echo >&2 "Please, fix them and re-run the provision script."
 219   exit 1
 220 fi
 221
 222 if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
 223   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
 224   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
 225   exit 1
 226 fi
 227
 228 # Only used in single_host/single_name deploys
 229 if [ "x${HOSTNAME_EXT}" = "x" ] ; then
 230   HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
 231 fi
 232
 233 if [ "${DUMP_CONFIG}" = "yes" ]; then
 234   echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
 235 else
 236   # Install a few dependency packages
 237   # First, let's figure out the OS we're working on
 238   OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d=  |cut -f 2 -d \")
 239   echo "Detected distro: ${OS_ID}"
 240
 241   case ${OS_ID} in
 242     "centos")
 243       yum install -y  curl git jq
 244       ;;
 245     "debian"|"ubuntu")
 246       DEBIAN_FRONTEND=noninteractive apt update
 247       DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
 248       ;;
 249   esac
 250
 251   if which salt-call; then
 252     echo "Salt already installed"
 253   else
 254     curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 255     sh /tmp/bootstrap_salt.sh -XdfP -x python3
 256     /bin/systemctl stop salt-minion.service
 257     /bin/systemctl disable salt-minion.service
 258   fi
 259
 260   # Set salt to masterless mode
 261   cat > /etc/salt/minion << EOFSM
 262 failhard: "True"
 263
 264 file_client: local
 265 file_roots:
 266   base:
 267     - ${S_DIR}
 268     - ${F_DIR}/*
 269
 270 pillar_roots:
 271   base:
 272     - ${P_DIR}
 273 EOFSM
 274 fi
 275
 276 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
 277
 278 # Get the formula and dependencies
 279 cd ${F_DIR} || exit 1
 280 echo "Cloning formulas"
 281 rm -rf ${F_DIR}/* || exit 1
 282 git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
 283 ( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
 284
 285 git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
 286 ( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
 287
 288 git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
 289 ( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
 290
 291 git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
 292 ( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
 293
 294 git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
 295 ( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
 296
 297 git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
 298
 299 # If we want to try a specific branch of the formula
 300 if [ "x${BRANCH}" != "x" ]; then
 301   ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
 302 elif [ "x${ARVADOS_TAG}" != "x" ]; then
 303 ( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
 304 fi
 305
 306 if [ "x${VAGRANT}" = "xyes" ]; then
 307   EXTRA_STATES_DIR="/home/vagrant/${CONFIG_DIR}/states"
 308   SOURCE_PILLARS_DIR="/home/vagrant/${CONFIG_DIR}/pillars"
 309   SOURCE_TESTS_DIR="/home/vagrant/${TESTS_DIR}"
 310 else
 311   EXTRA_STATES_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/states"
 312   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
 313   SOURCE_TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 314 fi
 315
 316 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
 317
 318 echo "Writing pillars and states"
 319
 320 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
 321 # to ease deployment for newcomers
 322 if [ ! -d "${SOURCE_PILLARS_DIR}" ]; then
 323   echo "${SOURCE_PILLARS_DIR} does not exist or is not a directory. Exiting."
 324   exit 1
 325 fi
 326 for f in $(ls "${SOURCE_PILLARS_DIR}"/*); do
 327   sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
 328        s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
 329        s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
 330        s#__CLUSTER__#${CLUSTER}#g;
 331        s#__DOMAIN__#${DOMAIN}#g;
 332        s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
 333        s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
 334        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
 335        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
 336        s#__INITIAL_USER__#${INITIAL_USER}#g;
 337        s#__LE_AWS_REGION__#${LE_AWS_REGION}#g;
 338        s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY}#g;
 339        s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID}#g;
 340        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
 341        s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
 342        s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
 343        s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g;
 344        s#__RELEASE__#${RELEASE}#g;
 345        s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g;
 346        s#__VERSION__#${VERSION}#g;
 347        s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
 348        s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
 349        s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
 350        s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
 351        s#__CLUSTER_INT_CIDR__#${CLUSTER_INT_CIDR}#g;
 352        s#__CONTROLLER_INT_IP__#${CONTROLLER_INT_IP}#g;
 353        s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
 354        s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
 355        s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
 356        s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
 357        s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
 358        s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
 359        s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
 360        s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
 361        s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
 362        s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
 363        s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
 364   "${f}" > "${P_DIR}"/$(basename "${f}")
 365 done
 366
 367 if [ "x${TEST}" = "xyes" ] && [ ! -d "${SOURCE_TESTS_DIR}" ]; then
 368   echo "You requested to run tests, but ${SOURCE_TESTS_DIR} does not exist or is not a directory. Exiting."
 369   exit 1
 370 fi
 371 mkdir -p ${T_DIR}
 372 # Replace cluster and domain name in the test files
 373 for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
 374   sed "s#__CLUSTER__#${CLUSTER}#g;
 375        s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
 376        s#__DOMAIN__#${DOMAIN}#g;
 377        s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
 378        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
 379        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g
 380        s#__INITIAL_USER__#${INITIAL_USER}#g;
 381        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
 382        s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
 383   "${f}" > ${T_DIR}/$(basename "${f}")
 384 done
 385 chmod 755 ${T_DIR}/run-test.sh
 386
 387 # Replace helper state files that differ from the formula's examples
 388 if [ -d "${SOURCE_STATES_DIR}" ]; then
 389   mkdir -p "${F_DIR}"/extra/extra
 390
 391   for f in $(ls "${SOURCE_STATES_DIR}"/*); do
 392     sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
 393          s#__CLUSTER__#${CLUSTER}#g;
 394          s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
 395          s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
 396          s#__DOMAIN__#${DOMAIN}#g;
 397          s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
 398          s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
 399          s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
 400          s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
 401          s#__INITIAL_USER__#${INITIAL_USER}#g;
 402          s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
 403          s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
 404          s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
 405          s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g;
 406          s#__RELEASE__#${RELEASE}#g;
 407          s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g;
 408          s#__VERSION__#${VERSION}#g;
 409          s#__CLUSTER_INT_CIDR__#${CLUSTER_INT_CIDR}#g;
 410          s#__CONTROLLER_INT_IP__#${CONTROLLER_INT_IP}#g;
 411          s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
 412          s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
 413          s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
 414          s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
 415          s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
 416          s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
 417          s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
 418          s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
 419          s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
 420          s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
 421          s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
 422          s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
 423          s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
 424          s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
 425     "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
 426   done
 427 fi
 428
 429 # Now, we build the SALT states/pillars trees
 430 # As we need to separate both states and pillars in case we want specific
 431 # roles, we iterate on both at the same time
 432
 433 # States
 434 cat > ${S_DIR}/top.sls << EOFTSLS
 435 base:
 436   '*':
 437     - locale
 438 EOFTSLS
 439
 440 # Pillars
 441 cat > ${P_DIR}/top.sls << EOFPSLS
 442 base:
 443   '*':
 444     - locale
 445     - arvados
 446 EOFPSLS
 447
 448 # States, extra states
 449 if [ -d "${F_DIR}"/extra/extra ]; then
 450   for f in $(ls "${F_DIR}"/extra/extra/*.sls); do
 451   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
 452   done
 453 fi
 454
 455 # If we want specific roles for a node, just add the desired states
 456 # and its dependencies
 457 if [ -z "${ROLES}" ]; then
 458   # States
 459   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 460   # Currently, only available on config_examples/multi_host/aws
 461   if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 462     if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 463       grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
 464     fi
 465     grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
 466   fi
 467   echo "    - postgres" >> ${S_DIR}/top.sls
 468   echo "    - docker.software" >> ${S_DIR}/top.sls
 469   echo "    - arvados" >> ${S_DIR}/top.sls
 470
 471   # Pillars
 472   echo "    - docker" >> ${P_DIR}/top.sls
 473   echo "    - nginx_api_configuration" >> ${P_DIR}/top.sls
 474   echo "    - nginx_controller_configuration" >> ${P_DIR}/top.sls
 475   echo "    - nginx_keepproxy_configuration" >> ${P_DIR}/top.sls
 476   echo "    - nginx_keepweb_configuration" >> ${P_DIR}/top.sls
 477   echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 478   echo "    - nginx_websocket_configuration" >> ${P_DIR}/top.sls
 479   echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 480   echo "    - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
 481   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
 482   echo "    - postgresql" >> ${P_DIR}/top.sls
 483   # Currently, only available on config_examples/multi_host/aws
 484   if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 485     if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 486       grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
 487     fi
 488     grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
 489   fi
 490 else
 491   # If we add individual roles, make sure we add the repo first
 492   echo "    - arvados.repo" >> ${S_DIR}/top.sls
 493   for R in ${ROLES}; do
 494     case "${R}" in
 495       "database")
 496         # States
 497         echo "    - postgres" >> ${S_DIR}/top.sls
 498         # Pillars
 499         echo '    - postgresql' >> ${P_DIR}/top.sls
 500       ;;
 501       "api")
 502         # States
 503         # FIXME: https://dev.arvados.org/issues/17352
 504         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
 505         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 506         ### If we don't install and run LE before arvados-api-server, it fails and breaks everything
 507         ### after it so we add this here, as we are, after all, sharing the host for api and controller
 508         # Currently, only available on config_examples/multi_host/aws
 509         if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 510           if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 511             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
 512           fi
 513           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
 514         fi
 515         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 516         # Pillars
 517         grep -q "aws_credentials" ${P_DIR}/top.sls          || echo "    - aws_credentials" >> ${P_DIR}/top.sls
 518         grep -q "docker" ${P_DIR}/top.sls                   || echo "    - docker" >> ${P_DIR}/top.sls
 519         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
 520         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 521         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 522       ;;
 523       "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
 524         # States
 525         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 526         # Currently, only available on config_examples/multi_host/aws
 527         if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 528           if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 529             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
 530           fi
 531           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
 532         fi
 533         # webshell role is just a nginx vhost, so it has no state
 534         if [ "${R}" != "webshell" ]; then
 535           grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 536         fi
 537         # Pillars
 538         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 539         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 540         # Currently, only available on config_examples/multi_host/aws
 541         if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 542           if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 543             grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
 544           fi
 545           grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
 546           grep -q "letsencrypt_${R}_configuration" ${P_DIR}/top.sls || echo "    - letsencrypt_${R}_configuration" >> ${P_DIR}/top.sls
 547         fi
 548       ;;
 549       "shell")
 550         # States
 551         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker.software" >> ${S_DIR}/top.sls
 552         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 553         # Pillars
 554         grep -q "" ${P_DIR}/top.sls                             || echo "    - docker" >> ${P_DIR}/top.sls
 555       ;;
 556       "dispatcher")
 557         # States
 558         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker.software" >> ${S_DIR}/top.sls
 559         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 560         # Pillars
 561         # ATM, no specific pillar needed
 562       ;;
 563       "keepstore")
 564         # States
 565         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 566         # Pillars
 567         # ATM, no specific pillar needed
 568       ;;
 569       *)
 570         echo "Unknown role ${R}"
 571         exit 1
 572       ;;
 573     esac
 574   done
 575 fi
 576
 577 if [ "${DUMP_CONFIG}" = "yes" ]; then
 578   # We won't run the rest of the script because we're just dumping the config
 579   exit 0
 580 fi
 581
 582 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 583 if [ -e /root/.psqlrc ]; then
 584   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 585     RESTORE_PSQL="yes"
 586     cp /root/.psqlrc /root/.psqlrc.provision.backup
 587   fi
 588 else
 589   DELETE_PSQL="yes"
 590 fi
 591
 592 echo '\pset pager off' >> /root/.psqlrc
 593 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 594
 595 # Now run the install
 596 salt-call --local state.apply -l ${LOG_LEVEL}
 597
 598 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 599 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 600   echo "Removing .psql file"
 601   rm /root/.psqlrc
 602 fi
 603
 604 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 605   echo "Restoring .psql file"
 606   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 607 fi
 608 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 609
 610 # Leave a copy of the Arvados CA so the user can copy it where it's required
 611 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 612 # If running in a vagrant VM, also add default user to docker group
 613 if [ "x${VAGRANT}" = "xyes" ]; then
 614   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 615
 616   echo "Adding the vagrant user to the docker group"
 617   usermod -a -G docker vagrant
 618 else
 619   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 620 fi
 621
 622 # Test that the installation finished correctly
 623 if [ "x${TEST}" = "xyes" ]; then
 624   cd ${T_DIR}
 625   # If we use RVM, we need to run this with it, or most ruby commands will fail
 626   RVM_EXEC=""
 627   if [ -x /usr/local/rvm/bin/rvm-exec ]; then
 628     RVM_EXEC="/usr/local/rvm/bin/rvm-exec"
 629   fi
 630   ${RVM_EXEC} ./run-test.sh
 631 fi