tools/salt-install/provision.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14 set -x
  15
  16 # capture the directory that the script is running from
  17 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  18
  19 usage() {
  20   echo >&2
  21   echo >&2 "Usage: ${0} [-h] [-h]"
  22   echo >&2
  23   echo >&2 "${0} options:"
  24   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  25   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  26   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  27   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  28   echo >&2 "                                              Possible values are:"
  29   echo >&2 "                                                api"
  30   echo >&2 "                                                controller"
  31   echo >&2 "                                                dispatcher"
  32   echo >&2 "                                                keepproxy"
  33   echo >&2 "                                                keepstore"
  34   echo >&2 "                                                keepweb"
  35   echo >&2 "                                                shell"
  36   echo >&2 "                                                webshell"
  37   echo >&2 "                                                websocket"
  38   echo >&2 "                                                workbench"
  39   echo >&2 "                                                workbench2"
  40   echo >&2 "                                              Defaults to applying them all"
  41   echo >&2 "  -h, --help                                  Display this help and exit"
  42   echo >&2 "  --dump-config <dest_dir>                    Dumps the pillars and states to a directory"
  43   echo >&2 "                                              This parameter does not perform any installation at all. It's"
  44   echo >&2 "                                              intended to give you a parsed sot of configuration files so"
  45   echo >&2 "                                              you can inspect them or use them in you Saltstack infrastructure."
  46   echo >&2 "                                              It"
  47   echo >&2 "                                                - parses the pillar and states templates,"
  48   echo >&2 "                                                - downloads the helper formulas with their desired versions,"
  49   echo >&2 "                                                - prepares the 'top.sls' files both for pillars and states"
  50   echo >&2 "                                                  for the selected role/s"
  51   echo >&2 "                                                - writes the resulting files into <dest_dir>"
  52   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  53   echo >&2
  54 }
  55
  56 arguments() {
  57   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  58   if ! which getopt > /dev/null; then
  59     echo >&2 "GNU getopt is required to run this script. Please install it and re-reun it"
  60     exit 1
  61   fi
  62
  63   TEMP=$(getopt -o c:dhp:r:tv \
  64     --long config:,debug,dump-config:,help,roles:,test,vagrant \
  65     -n "${0}" -- "${@}")
  66
  67   if [ ${?} != 0 ];
  68     then echo "Please check the parameters you entered and re-run again"
  69     exit 1
  70   fi
  71   # Note the quotes around `$TEMP': they are essential!
  72   eval set -- "$TEMP"
  73
  74   while [ ${#} -ge 1 ]; do
  75     case ${1} in
  76       -c | --config)
  77         CONFIG_FILE=${2}
  78         shift 2
  79         ;;
  80       -d | --debug)
  81         LOG_LEVEL="debug"
  82         shift
  83         set -x
  84         ;;
  85       --dump-config)
  86         if [[ ${2} = /* ]]; then
  87           DUMP_SALT_CONFIG_DIR=${2}
  88         else
  89           DUMP_SALT_CONFIG_DIR=${PWD}/${2}
  90         fi
  91         ## states
  92         S_DIR="${DUMP_SALT_CONFIG_DIR}/salt"
  93         ## formulas
  94         F_DIR="${DUMP_SALT_CONFIG_DIR}/formulas"
  95         ## pillars
  96         P_DIR="${DUMP_SALT_CONFIG_DIR}/pillars"
  97         ## tests
  98         T_DIR="${DUMP_SALT_CONFIG_DIR}/tests"
  99         DUMP_CONFIG="yes"
 100         shift 2
 101         ;;
 102       -r | --roles)
 103         for i in ${2//,/ }
 104           do
 105             # Verify the role exists
 106             if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
 107               echo "The role '${i}' is not a valid role"
 108               usage
 109               exit 1
 110             fi
 111             ROLES="${ROLES} ${i}"
 112           done
 113           shift 2
 114         ;;
 115       -t | --test)
 116         TEST="yes"
 117         shift
 118         ;;
 119       -v | --vagrant)
 120         VAGRANT="yes"
 121         shift
 122         ;;
 123       --)
 124         shift
 125         break
 126         ;;
 127       *)
 128         usage
 129         exit 1
 130         ;;
 131     esac
 132   done
 133 }
 134
 135 CONFIG_FILE="${SCRIPT_DIR}/local.params"
 136 CONFIG_DIR="local_config_dir"
 137 DUMP_CONFIG="no"
 138 LOG_LEVEL="info"
 139 CONTROLLER_EXT_SSL_PORT=443
 140 TESTS_DIR="tests"
 141
 142 CLUSTER=""
 143 DOMAIN=""
 144
 145 # Hostnames/IPs used for single-host deploys
 146 HOSTNAME_EXT=""
 147 HOSTNAME_INT="127.0.1.1"
 148
 149 # Initial user setup
 150 INITIAL_USER=""
 151 INITIAL_USER_EMAIL=""
 152 INITIAL_USER_PASSWORD=""
 153
 154 CONTROLLER_EXT_SSL_PORT=8000
 155 KEEP_EXT_SSL_PORT=25101
 156 # Both for collections and downloads
 157 KEEPWEB_EXT_SSL_PORT=9002
 158 WEBSHELL_EXT_SSL_PORT=4202
 159 WEBSOCKET_EXT_SSL_PORT=8002
 160 WORKBENCH1_EXT_SSL_PORT=443
 161 WORKBENCH2_EXT_SSL_PORT=3001
 162
 163 ## These are ARVADOS-related parameters
 164 # For a stable release, change RELEASE "production" and VERSION to the
 165 # package version (including the iteration, e.g. X.Y.Z-1) of the
 166 # release.
 167 # The "local.params.example.*" files already set "RELEASE=production"
 168 # to deploy  production-ready packages
 169 RELEASE="development"
 170 VERSION="latest"
 171
 172 # These are arvados-formula-related parameters
 173 # An arvados-formula tag. For a stable release, this should be a
 174 # branch name (e.g. X.Y-dev) or tag for the release.
 175 # ARVADOS_TAG="2.2.0"
 176 # BRANCH="main"
 177
 178 # Other formula versions we depend on
 179 POSTGRES_TAG="v0.41.6"
 180 NGINX_TAG="temp-fix-missing-statements-in-pillar"
 181 DOCKER_TAG="v2.0.7"
 182 LOCALE_TAG="v0.3.4"
 183 LETSENCRYPT_TAG="v2.1.0"
 184
 185 # Salt's dir
 186 DUMP_SALT_CONFIG_DIR=""
 187 ## states
 188 S_DIR="/srv/salt"
 189 ## formulas
 190 F_DIR="/srv/formulas"
 191 ## pillars
 192 P_DIR="/srv/pillars"
 193 ## tests
 194 T_DIR="/tmp/cluster_tests"
 195
 196 arguments ${@}
 197
 198 if [ -s ${CONFIG_FILE} ]; then
 199   source ${CONFIG_FILE}
 200 else
 201   echo >&2 "You don't seem to have a config file with initial values."
 202   echo >&2 "Please create a '${CONFIG_FILE}' file as described in"
 203   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
 204   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
 205   exit 1
 206 fi
 207
 208 if [ ! -d ${CONFIG_DIR} ]; then
 209   echo >&2 "You don't seem to have a config directory with pillars and states."
 210   echo >&2 "Please create a '${CONFIG_DIR}' directory (as configured in your '${CONFIG_FILE}'). Please see"
 211   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
 212   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
 213   exit 1
 214 fi
 215
 216 if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
 217   echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified."
 218   echo >&2 "Please, fix them and re-run the provision script."
 219   exit 1
 220 fi
 221
 222 if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
 223   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
 224   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
 225   exit 1
 226 fi
 227
 228 # Only used in single_host/single_name deploys
 229 if [ "x${HOSTNAME_EXT}" = "x" ] ; then
 230   HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
 231 fi
 232
 233 if [ "${DUMP_CONFIG}" = "yes" ]; then
 234   echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
 235 else
 236   # Install a few dependency packages
 237   # First, let's figure out the OS we're working on
 238   OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d=  |cut -f 2 -d \")
 239   echo "Detected distro: ${OS_ID}"
 240
 241   case ${OS_ID} in
 242     "centos")
 243       yum install -y  curl git jq
 244       ;;
 245     "debian"|"ubuntu")
 246       DEBIAN_FRONTEND=noninteractive apt update
 247       DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
 248       ;;
 249   esac
 250
 251   if which salt-call; then
 252     echo "Salt already installed"
 253   else
 254     curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 255     sh /tmp/bootstrap_salt.sh -XdfP -x python3
 256     /bin/systemctl stop salt-minion.service
 257     /bin/systemctl disable salt-minion.service
 258   fi
 259
 260   # Set salt to masterless mode
 261   cat > /etc/salt/minion << EOFSM
 262 file_client: local
 263 file_roots:
 264   base:
 265     - ${S_DIR}
 266     - ${F_DIR}/*
 267
 268 pillar_roots:
 269   base:
 270     - ${P_DIR}
 271 EOFSM
 272 fi
 273
 274 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
 275
 276 # Get the formula and dependencies
 277 cd ${F_DIR} || exit 1
 278 echo "Cloning formulas"
 279 rm -rf ${F_DIR}/* || exit 1
 280 git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
 281 ( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
 282
 283 git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
 284 ( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
 285
 286 git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
 287 ( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
 288
 289 git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
 290 ( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
 291
 292 git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
 293 ( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
 294
 295 git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
 296
 297 # If we want to try a specific branch of the formula
 298 if [ "x${BRANCH}" != "x" ]; then
 299   ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
 300 elif [ "x${ARVADOS_TAG}" != "x" ]; then
 301 ( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
 302 fi
 303
 304 if [ "x${VAGRANT}" = "xyes" ]; then
 305   EXTRA_STATES_DIR="/home/vagrant/${CONFIG_DIR}/states"
 306   SOURCE_PILLARS_DIR="/home/vagrant/${CONFIG_DIR}/pillars"
 307   SOURCE_TESTS_DIR="/home/vagrant/${TESTS_DIR}"
 308 else
 309   EXTRA_STATES_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/states"
 310   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
 311   SOURCE_TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 312 fi
 313
 314 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
 315
 316 echo "Writing pillars and states"
 317
 318 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
 319 # to ease deployment for newcomers
 320 if [ ! -d "${SOURCE_PILLARS_DIR}" ]; then
 321   echo "${SOURCE_PILLARS_DIR} does not exist or is not a directory. Exiting."
 322   exit 1
 323 fi
 324 for f in $(ls "${SOURCE_PILLARS_DIR}"/*); do
 325   sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
 326        s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
 327        s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
 328        s#__CLUSTER__#${CLUSTER}#g;
 329        s#__DOMAIN__#${DOMAIN}#g;
 330        s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
 331        s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
 332        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
 333        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
 334        s#__INITIAL_USER__#${INITIAL_USER}#g;
 335        s#__LE_AWS_REGION__#${LE_AWS_REGION}#g;
 336        s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY}#g;
 337        s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID}#g;
 338        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
 339        s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
 340        s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
 341        s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g;
 342        s#__RELEASE__#${RELEASE}#g;
 343        s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g;
 344        s#__VERSION__#${VERSION}#g;
 345        s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
 346        s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
 347        s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
 348        s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
 349        s#__CLUSTER_INT_CIDR__#${CLUSTER_INT_CIDR}#g;
 350        s#__CONTROLLER_INT_IP__#${CONTROLLER_INT_IP}#g;
 351        s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
 352        s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
 353        s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
 354        s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
 355        s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
 356        s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
 357        s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
 358        s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
 359        s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
 360        s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
 361        s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
 362   "${f}" > "${P_DIR}"/$(basename "${f}")
 363 done
 364
 365 if [ "x${TEST}" = "xyes" ] && [ ! -d "${SOURCE_TESTS_DIR}" ]; then
 366   echo "You requested to run tests, but ${SOURCE_TESTS_DIR} does not exist or is not a directory. Exiting."
 367   exit 1
 368 fi
 369 mkdir -p ${T_DIR}
 370 # Replace cluster and domain name in the test files
 371 for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
 372   sed "s#__CLUSTER__#${CLUSTER}#g;
 373        s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
 374        s#__DOMAIN__#${DOMAIN}#g;
 375        s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
 376        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
 377        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g
 378        s#__INITIAL_USER__#${INITIAL_USER}#g;
 379        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
 380        s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
 381   "${f}" > ${T_DIR}/$(basename "${f}")
 382 done
 383 chmod 755 ${T_DIR}/run-test.sh
 384
 385 # Replace helper state files that differ from the formula's examples
 386 if [ -d "${SOURCE_STATES_DIR}" ]; then
 387   mkdir -p "${F_DIR}"/extra/extra
 388
 389   for f in $(ls "${SOURCE_STATES_DIR}"/*); do
 390     sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
 391          s#__CLUSTER__#${CLUSTER}#g;
 392          s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
 393          s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
 394          s#__DOMAIN__#${DOMAIN}#g;
 395          s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
 396          s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
 397          s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
 398          s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
 399          s#__INITIAL_USER__#${INITIAL_USER}#g;
 400          s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
 401          s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
 402          s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
 403          s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g;
 404          s#__RELEASE__#${RELEASE}#g;
 405          s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g;
 406          s#__VERSION__#${VERSION}#g;
 407          s#__CLUSTER_INT_CIDR__#${CLUSTER_INT_CIDR}#g;
 408          s#__CONTROLLER_INT_IP__#${CONTROLLER_INT_IP}#g;
 409          s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
 410          s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
 411          s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
 412          s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
 413          s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
 414          s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
 415          s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
 416          s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
 417          s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
 418          s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
 419          s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
 420          s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
 421          s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
 422          s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
 423     "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
 424   done
 425 fi
 426
 427 # Now, we build the SALT states/pillars trees
 428 # As we need to separate both states and pillars in case we want specific
 429 # roles, we iterate on both at the same time
 430
 431 # States
 432 cat > ${S_DIR}/top.sls << EOFTSLS
 433 base:
 434   '*':
 435     - locale
 436 EOFTSLS
 437
 438 # Pillars
 439 cat > ${P_DIR}/top.sls << EOFPSLS
 440 base:
 441   '*':
 442     - locale
 443     - arvados
 444 EOFPSLS
 445
 446 # States, extra states
 447 if [ -d "${F_DIR}"/extra/extra ]; then
 448   for f in $(ls "${F_DIR}"/extra/extra/*.sls); do
 449   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
 450   done
 451 fi
 452
 453 # If we want specific roles for a node, just add the desired states
 454 # and its dependencies
 455 if [ -z "${ROLES}" ]; then
 456   # States
 457   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 458   # Currently, only available on config_examples/multi_host/aws
 459   if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 460     if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 461       grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
 462     fi
 463     grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
 464   fi
 465   echo "    - postgres" >> ${S_DIR}/top.sls
 466   echo "    - docker.software" >> ${S_DIR}/top.sls
 467   echo "    - arvados" >> ${S_DIR}/top.sls
 468
 469   # Pillars
 470   echo "    - docker" >> ${P_DIR}/top.sls
 471   echo "    - nginx_api_configuration" >> ${P_DIR}/top.sls
 472   echo "    - nginx_controller_configuration" >> ${P_DIR}/top.sls
 473   echo "    - nginx_keepproxy_configuration" >> ${P_DIR}/top.sls
 474   echo "    - nginx_keepweb_configuration" >> ${P_DIR}/top.sls
 475   echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 476   echo "    - nginx_websocket_configuration" >> ${P_DIR}/top.sls
 477   echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 478   echo "    - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
 479   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
 480   echo "    - postgresql" >> ${P_DIR}/top.sls
 481   # Currently, only available on config_examples/multi_host/aws
 482   if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 483     if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 484       grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
 485     fi
 486     grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
 487   fi
 488 else
 489   # If we add individual roles, make sure we add the repo first
 490   echo "    - arvados.repo" >> ${S_DIR}/top.sls
 491   for R in ${ROLES}; do
 492     case "${R}" in
 493       "database")
 494         # States
 495         echo "    - postgres" >> ${S_DIR}/top.sls
 496         # Pillars
 497         echo '    - postgresql' >> ${P_DIR}/top.sls
 498       ;;
 499       "api")
 500         # States
 501         # FIXME: https://dev.arvados.org/issues/17352
 502         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
 503         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 504         ### If we don't install and run LE before arvados-api-server, it fails and breaks everything
 505         ### after it so we add this here, as we are, after all, sharing the host for api and controller
 506         # Currently, only available on config_examples/multi_host/aws
 507         if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 508           if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 509             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
 510           fi
 511           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
 512         fi
 513         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 514         # Pillars
 515         grep -q "aws_credentials" ${P_DIR}/top.sls          || echo "    - aws_credentials" >> ${P_DIR}/top.sls
 516         grep -q "docker" ${P_DIR}/top.sls                   || echo "    - docker" >> ${P_DIR}/top.sls
 517         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
 518         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 519         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 520       ;;
 521       "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
 522         # States
 523         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 524         # Currently, only available on config_examples/multi_host/aws
 525         if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 526           if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 527             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
 528           fi
 529           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
 530         fi
 531         # webshell role is just a nginx vhost, so it has no state
 532         if [ "${R}" != "webshell" ]; then
 533           grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 534         fi
 535         # Pillars
 536         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 537         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 538         # Currently, only available on config_examples/multi_host/aws
 539         if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
 540           if [ "x${USE_LETSENCRYPT_IAM_USER}" = "xyes" ]; then
 541             grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
 542           fi
 543           grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
 544           grep -q "letsencrypt_${R}_configuration" ${P_DIR}/top.sls || echo "    - letsencrypt_${R}_configuration" >> ${P_DIR}/top.sls
 545         fi
 546       ;;
 547       "shell")
 548         # States
 549         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker.software" >> ${S_DIR}/top.sls
 550         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 551         # Pillars
 552         grep -q "" ${P_DIR}/top.sls                             || echo "    - docker" >> ${P_DIR}/top.sls
 553       ;;
 554       "dispatcher")
 555         # States
 556         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker.software" >> ${S_DIR}/top.sls
 557         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 558         # Pillars
 559         # ATM, no specific pillar needed
 560       ;;
 561       "keepstore")
 562         # States
 563         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 564         # Pillars
 565         # ATM, no specific pillar needed
 566       ;;
 567       *)
 568         echo "Unknown role ${R}"
 569         exit 1
 570       ;;
 571     esac
 572   done
 573 fi
 574
 575 if [ "${DUMP_CONFIG}" = "yes" ]; then
 576   # We won't run the rest of the script because we're just dumping the config
 577   exit 0
 578 fi
 579
 580 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 581 if [ -e /root/.psqlrc ]; then
 582   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 583     RESTORE_PSQL="yes"
 584     cp /root/.psqlrc /root/.psqlrc.provision.backup
 585   fi
 586 else
 587   DELETE_PSQL="yes"
 588 fi
 589
 590 echo '\pset pager off' >> /root/.psqlrc
 591 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 592
 593 # Now run the install
 594 salt-call --local state.apply -l ${LOG_LEVEL}
 595
 596 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 597 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 598   echo "Removing .psql file"
 599   rm /root/.psqlrc
 600 fi
 601
 602 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 603   echo "Restoring .psql file"
 604   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 605 fi
 606 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 607
 608 # Leave a copy of the Arvados CA so the user can copy it where it's required
 609 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 610 # If running in a vagrant VM, also add default user to docker group
 611 if [ "x${VAGRANT}" = "xyes" ]; then
 612   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 613
 614   echo "Adding the vagrant user to the docker group"
 615   usermod -a -G docker vagrant
 616 else
 617   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 618 fi
 619
 620 # Test that the installation finished correctly
 621 if [ "x${TEST}" = "xyes" ]; then
 622   cd ${T_DIR}
 623   # If we use RVM, we need to run this with it, or most ruby commands will fail
 624   RVM_EXEC=""
 625   if [ -x /usr/local/rvm/bin/rvm-exec ]; then
 626     RVM_EXEC="/usr/local/rvm/bin/rvm-exec"
 627   fi
 628   ${RVM_EXEC} ./run-test.sh
 629 fi