tools/salt-install/provision.sh

   1 #!/bin/bash -x
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14
  15 # capture the directory that the script is running from
  16 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  17
  18 usage() {
  19   echo >&2
  20   echo >&2 "Usage: ${0} [-h] [-h]"
  21   echo >&2
  22   echo >&2 "${0} options:"
  23   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  24   echo >&2 "  -p <N>, --ssl-port <N>                      SSL port to use for the web applications"
  25   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  26   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  27   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  28   echo >&2 "                                              Possible values are:"
  29   echo >&2 "                                                api"
  30   echo >&2 "                                                controller"
  31   echo >&2 "                                                keepstore"
  32   echo >&2 "                                                websocket"
  33   echo >&2 "                                                keepweb"
  34   echo >&2 "                                                workbench2"
  35   echo >&2 "                                                keepproxy"
  36   echo >&2 "                                                shell"
  37   echo >&2 "                                                workbench"
  38   echo >&2 "                                                dispatcher"
  39   echo >&2 "                                              Defaults to applying them all"
  40   echo >&2 "  -h, --help                                  Display this help and exit"
  41   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  42   echo >&2
  43 }
  44
  45 arguments() {
  46   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  47   TEMP=$(getopt -o c:dhp:r:tv \
  48     --long config:,debug,help,ssl-port:,roles:,test,vagrant \
  49     -n "${0}" -- "${@}")
  50
  51   if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
  52   # Note the quotes around `$TEMP': they are essential!
  53   eval set -- "$TEMP"
  54
  55   while [ ${#} -ge 1 ]; do
  56     case ${1} in
  57       -c | --config)
  58         CONFIG_FILE=${2}
  59         shift 2
  60         ;;
  61       -d | --debug)
  62         LOG_LEVEL="debug"
  63         shift
  64         ;;
  65       -p | --ssl-port)
  66         CONTROLLER_EXT_SSL_PORT=${2}
  67         shift 2
  68         ;;
  69       -r | --roles)
  70         for i in ${2//,/ }
  71           do
  72             # Verify the role exists
  73             if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
  74               echo "The role '${i}' is not a valid role"
  75               usage
  76               exit 1
  77             fi
  78             ROLES="${ROLES} ${i}"
  79           done
  80           shift 2
  81         ;;
  82       -t | --test)
  83         TEST="yes"
  84         shift
  85         ;;
  86       -v | --vagrant)
  87         VAGRANT="yes"
  88         shift
  89         ;;
  90       --)
  91         shift
  92         break
  93         ;;
  94       *)
  95         usage
  96         exit 1
  97         ;;
  98     esac
  99   done
 100 }
 101
 102 CONFIG="${SCRIPT_DIR}/local.params"
 103 CONFIG_DIR="config_examples/single_host/multiple_hostnames"
 104 LOG_LEVEL="info"
 105 CONTROLLER_EXT_SSL_PORT=443
 106 TESTS_DIR="tests"
 107
 108 CLUSTER=""
 109 DOMAIN=""
 110
 111 # Hostnames/IPs used for single-host deploys
 112 HOSTNAME_EXT=""
 113 HOSTNAME_INT="127.0.1.1"
 114
 115 # Initial user setup
 116 INITIAL_USER=""
 117 INITIAL_USER_EMAIL=""
 118 INITIAL_USER_PASSWORD=""
 119
 120 CONTROLLER_EXT_SSL_PORT=8000
 121 KEEP_EXT_SSL_PORT=25101
 122 # Both for collections and downloads
 123 KEEPWEB_EXT_SSL_PORT=9002
 124 WEBSHELL_EXT_SSL_PORT=4202
 125 WEBSOCKET_EXT_SSL_PORT=8002
 126 WORKBENCH1_EXT_SSL_PORT=443
 127 WORKBENCH2_EXT_SSL_PORT=3001
 128
 129 RELEASE="production"
 130 VERSION="latest"
 131 ARVADOS_TAG="v1.1.4"
 132 POSTGRES_TAG="v0.41.3"
 133 NGINX_TAG="v2.4.0"
 134 DOCKER_TAG="v1.0.0"
 135 LOCALE_TAG="v0.3.4"
 136
 137 # Salt's dir
 138 ## states
 139 S_DIR="/srv/salt"
 140 ## formulas
 141 F_DIR="/srv/formulas"
 142 ##pillars
 143 P_DIR="/srv/pillars"
 144
 145 arguments ${@}
 146
 147 if [ -s ${CONFIG_FILE} ]; then
 148   source ${CONFIG_FILE}
 149 else
 150   echo >&2 "Please create a '${CONFIG_FILE}' file with initial values, as described in FIXME_URL_TO_DESCR"
 151   exit 1
 152 fi
 153
 154 if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
 155   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
 156   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
 157   exit 1
 158 fi
 159
 160 apt-get update
 161 apt-get install -y curl git jq
 162
 163 if which salt-call; then
 164   echo "Salt already installed"
 165 else
 166   curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 167   sh /tmp/bootstrap_salt.sh -XdfP -x python3
 168   /bin/systemctl stop salt-minion.service
 169   /bin/systemctl disable salt-minion.service
 170 fi
 171
 172 # Set salt to masterless mode
 173 cat > /etc/salt/minion << EOFSM
 174 file_client: local
 175 file_roots:
 176   base:
 177     - ${S_DIR}
 178     - ${F_DIR}/*
 179
 180 pillar_roots:
 181   base:
 182     - ${P_DIR}
 183 EOFSM
 184
 185 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
 186
 187 # Get the formula and dependencies
 188 cd ${F_DIR} || exit 1
 189 git clone --branch "${ARVADOS_TAG}" https://github.com/arvados/arvados-formula.git
 190 git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
 191 git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
 192 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 193 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 194
 195 if [ "x${BRANCH}" != "x" ]; then
 196   cd ${F_DIR}/arvados-formula || exit 1
 197   git checkout -t origin/"${BRANCH}"
 198   cd -
 199 fi
 200
 201 if [ "x${VAGRANT}" = "xyes" ]; then
 202   SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}/pillars"
 203   SOURCE_TESTS_DIR="/vagrant/${TESTS_DIR}"
 204 else
 205   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
 206   SOURCE_TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 207 fi
 208
 209 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
 210
 211 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
 212 # to ease deployment for newcomers
 213 for f in "${SOURCE_PILLARS_DIR}"/*; do
 214   sed "s/__ANONYMOUS_USER_TOKEN__/${ANONYMOUS_USER_TOKEN}/g;
 215        s/__BLOB_SIGNING_KEY__/${BLOB_SIGNING_KEY}/g;
 216        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 217        s/__CLUSTER__/${CLUSTER}/g;
 218        s/__DOMAIN__/${DOMAIN}/g;
 219        s/__HOSTNAME_EXT__/${HOSTNAME_EXT}/g;
 220        s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 221        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 222        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 223        s/__INITIAL_USER__/${INITIAL_USER}/g;
 224        s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 225        s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 226        s/__MANAGEMENT_TOKEN__/${MANAGEMENT_TOKEN}/g;
 227        s/__RELEASE__/${RELEASE}/g;
 228        s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g;
 229        s/__VERSION__/${VERSION}/g;
 230        s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 231        s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 232        s/__WORKBENCH1_EXT_SSL_PORT__/${WORKBENCH1_EXT_SSL_PORT}/g;
 233        s/__WORKBENCH2_EXT_SSL_PORT__/${WORKBENCH2_EXT_SSL_PORT}/g;
 234        s/__WORKBENCH_SECRET_KEY__/${WORKBENCH_SECRET_KEY}/g" \
 235   "${f}" > "${P_DIR}"/$(basename "${f}")
 236 done
 237
 238 mkdir -p /tmp/cluster_tests
 239 # Replace cluster and domain name in the test files
 240 for f in "${SOURCE_TESTS_DIR}"/*; do
 241   sed "s/__CLUSTER__/${CLUSTER}/g;
 242        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 243        s/__DOMAIN__/${DOMAIN}/g;
 244        s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 245        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 246        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g
 247        s/__INITIAL_USER__/${INITIAL_USER}/g;
 248        s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g" \
 249   "${f}" > "/tmp/cluster_tests"/$(basename "${f}")
 250 done
 251 chmod 755 /tmp/cluster_tests/run-test.sh
 252
 253 # Replace helper state files that differ from the formula's examples
 254 if [ -d "${SOURCE_STATES_DIR}" ]; then
 255   mkdir -p "${F_DIR}"/extra/extra
 256
 257   for f in "${SOURCE_STATES_DIR}"/*; do
 258     sed "s/__ANONYMOUS_USER_TOKEN__/${ANONYMOUS_USER_TOKEN}/g;
 259          s/__CLUSTER__/${CLUSTER}/g;
 260          s/__BLOB_SIGNING_KEY__/${BLOB_SIGNING_KEY}/g;
 261          s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 262          s/__DOMAIN__/${DOMAIN}/g;
 263          s/__HOSTNAME_EXT__/${HOSTNAME_EXT}/g;
 264          s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 265          s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 266          s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 267          s/__INITIAL_USER__/${INITIAL_USER}/g;
 268          s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 269          s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 270          s/__MANAGEMENT_TOKEN__/${MANAGEMENT_TOKEN}/g;
 271          s/__RELEASE__/${RELEASE}/g;
 272          s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g;
 273          s/__VERSION__/${VERSION}/g;
 274          s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 275          s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 276          s/__WORKBENCH1_EXT_SSL_PORT__/${WORKBENCH1_EXT_SSL_PORT}/g;
 277          s/__WORKBENCH2_EXT_SSL_PORT__/${WORKBENCH2_EXT_SSL_PORT}/g;
 278          s/__WORKBENCH_SECRET_KEY__/${WORKBENCH_SECRET_KEY}/g" \
 279     "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
 280   done
 281 fi
 282
 283 # Now, we build the SALT states/pillars trees
 284 # As we need to separate both states and pillars in case we want specific
 285 # roles, we iterate on both at the same time
 286
 287 # States
 288 cat > ${S_DIR}/top.sls << EOFTSLS
 289 base:
 290   '*':
 291     - locale
 292 EOFTSLS
 293
 294 # Pillars
 295 cat > ${P_DIR}/top.sls << EOFPSLS
 296 base:
 297   '*':
 298     - locale
 299     - arvados
 300 EOFPSLS
 301
 302 # States, extra states
 303 if [ -d "${F_DIR}"/extra/extra ]; then
 304   for f in "${F_DIR}"/extra/extra/*.sls; do
 305   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
 306   done
 307 fi
 308
 309 # If we want specific roles for a node, just add the desired states
 310 # and its dependencies
 311 if [ -z "${ROLES}" ]; then
 312   # States
 313   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 314   echo "    - postgres" >> ${S_DIR}/top.sls
 315   echo "    - docker" >> ${S_DIR}/top.sls
 316   echo "    - arvados" >> ${S_DIR}/top.sls
 317
 318   # Pillars
 319   echo "    - docker" >> ${P_DIR}/top.sls
 320   echo "    - nginx_api_configuration" >> ${P_DIR}/top.sls
 321   echo "    - nginx_controller_configuration" >> ${P_DIR}/top.sls
 322   echo "    - nginx_keepproxy_configuration" >> ${P_DIR}/top.sls
 323   echo "    - nginx_keepweb_configuration" >> ${P_DIR}/top.sls
 324   echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 325   echo "    - nginx_websocket_configuration" >> ${P_DIR}/top.sls
 326   echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 327   echo "    - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
 328   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
 329   echo "    - postgresql" >> ${P_DIR}/top.sls
 330 else
 331   # If we add individual roles, make sure we add the repo first
 332   echo "    - arvados.repo" >> ${S_DIR}/top.sls
 333   for R in ${ROLES}; do
 334     case "${R}" in
 335       "database")
 336         # States
 337         echo "    - postgres" >> ${S_DIR}/top.sls
 338         # Pillars
 339         echo '    - postgresql' >> ${P_DIR}/top.sls
 340       ;;
 341       "api")
 342         # States
 343         # FIXME: https://dev.arvados.org/issues/17352
 344         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
 345         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 346         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 347         # Pillars
 348         grep -q "docker" ${P_DIR}/top.sls                   || echo "    - docker" >> ${P_DIR}/top.sls
 349         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
 350         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 351         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 352       ;;
 353       "controller" | "websocket" | "workbench" | "workbench2" | "keepweb" | "keepproxy")
 354         # States
 355         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 356         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 357         # Pillars
 358         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 359         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 360       ;;
 361       "shell")
 362         # States
 363         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker" >> ${S_DIR}/top.sls
 364         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 365         # Pillars
 366         grep -q "" ${P_DIR}/top.sls                             || echo "    - docker" >> ${P_DIR}/top.sls
 367         grep -q "nginx_webshell_configuration" ${P_DIR}/top.sls || echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 368       ;;
 369       "dispatcher")
 370         # States
 371         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker" >> ${S_DIR}/top.sls
 372         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 373         # Pillars
 374         # ATM, no specific pillar needed
 375       ;;
 376       "keepstore")
 377         # States
 378         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 379         # Pillars
 380         # ATM, no specific pillar needed
 381       ;;
 382       *)
 383         echo "Unknown role ${R}"
 384         exit 1
 385       ;;
 386     esac
 387   done
 388 fi
 389
 390 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 391 if [ -e /root/.psqlrc ]; then
 392   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 393     RESTORE_PSQL="yes"
 394     cp /root/.psqlrc /root/.psqlrc.provision.backup
 395   fi
 396 else
 397   DELETE_PSQL="yes"
 398 fi
 399
 400 echo '\pset pager off' >> /root/.psqlrc
 401 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 402
 403 # Now run the install
 404 salt-call --local state.apply -l ${LOG_LEVEL}
 405
 406 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 407 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 408   echo "Removing .psql file"
 409   rm /root/.psqlrc
 410 fi
 411
 412 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 413   echo "Restoring .psql file"
 414   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 415 fi
 416 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 417
 418 # Leave a copy of the Arvados CA so the user can copy it where it's required
 419 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 420 # If running in a vagrant VM, also add default user to docker group
 421 if [ "x${VAGRANT}" = "xyes" ]; then
 422   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 423
 424   echo "Adding the vagrant user to the docker group"
 425   usermod -a -G docker vagrant
 426 else
 427   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 428 fi
 429
 430 # Test that the installation finished correctly
 431 if [ "x${TEST}" = "xyes" ]; then
 432   cd /tmp/cluster_tests
 433   ./run-test.sh
 434 fi