tools/salt-install/provision.sh

   1 #!/bin/bash -x
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14
  15 # capture the directory that the script is running from
  16 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  17
  18 usage() {
  19   echo >&2
  20   echo >&2 "Usage: ${0} [-h] [-h]"
  21   echo >&2
  22   echo >&2 "${0} options:"
  23   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  24   echo >&2 "  -p <N>, --ssl-port <N>                      SSL port to use for the web applications"
  25   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  26   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  27   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  28   echo >&2 "                                              Possible values are:"
  29   echo >&2 "                                                api"
  30   echo >&2 "                                                controller"
  31   echo >&2 "                                                keepstore"
  32   echo >&2 "                                                websocket"
  33   echo >&2 "                                                keepweb"
  34   echo >&2 "                                                workbench2"
  35   echo >&2 "                                                keepproxy"
  36   echo >&2 "                                                shell"
  37   echo >&2 "                                                workbench"
  38   echo >&2 "                                                dispatcher"
  39   echo >&2 "                                              Defaults to applying them all"
  40   echo >&2 "  -h, --help                                  Display this help and exit"
  41   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  42   echo >&2
  43 }
  44
  45 arguments() {
  46   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  47   TEMP=$(getopt -o c:dhp:r:tv \
  48     --long config:,debug,help,ssl-port:,roles:,test,vagrant \
  49     -n "${0}" -- "${@}")
  50
  51   if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
  52   # Note the quotes around `$TEMP': they are essential!
  53   eval set -- "$TEMP"
  54
  55   while [ ${#} -ge 1 ]; do
  56     case ${1} in
  57       -c | --config)
  58         CONFIG_FILE=${2}
  59         shift 2
  60         ;;
  61       -d | --debug)
  62         LOG_LEVEL="debug"
  63         shift
  64         ;;
  65       -p | --ssl-port)
  66         CONTROLLER_EXT_SSL_PORT=${2}
  67         shift 2
  68         ;;
  69       -r | --roles)
  70         for i in ${2//,/ }
  71           do
  72             # Verify the role exists
  73             if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
  74               echo "The role '${i}' is not a valid role"
  75               usage
  76               exit 1
  77             fi
  78             ROLES="${ROLES} ${i}"
  79           done
  80           shift 2
  81         ;;
  82       -t | --test)
  83         TEST="yes"
  84         shift
  85         ;;
  86       -v | --vagrant)
  87         VAGRANT="yes"
  88         shift
  89         ;;
  90       --)
  91         shift
  92         break
  93         ;;
  94       *)
  95         usage
  96         exit 1
  97         ;;
  98     esac
  99   done
 100 }
 101
 102 CONFIG="${SCRIPT_DIR}/local.params"
 103 CONFIG_DIR="config_examples/single_host/multiple_hostnames"
 104 LOG_LEVEL="info"
 105 CONTROLLER_EXT_SSL_PORT=443
 106 TESTS_DIR="tests"
 107
 108 CLUSTER=""
 109 DOMAIN=""
 110 HOSTNAME_EXT=""
 111 HOSTNAME_INT="127.0.1.1"
 112 INITIAL_USER=""
 113 INITIAL_USER_EMAIL=""
 114 INITIAL_USER_PASSWORD=""
 115
 116 CONTROLLER_EXT_SSL_PORT=8000
 117 KEEP_EXT_SSL_PORT=25101
 118 # Both for collections and downloads
 119 KEEPWEB_EXT_SSL_PORT=9002
 120 WEBSHELL_EXT_SSL_PORT=4202
 121 WEBSOCKET_EXT_SSL_PORT=8002
 122 WORKBENCH1_EXT_SSL_PORT=443
 123 WORKBENCH2_EXT_SSL_PORT=3001
 124
 125 RELEASE="production"
 126 VERSION="latest"
 127 ARVADOS_TAG="v1.1.4"
 128 POSTGRES_TAG="v0.41.3"
 129 NGINX_TAG="v2.4.0"
 130 DOCKER_TAG="v1.0.0"
 131 LOCALE_TAG="v0.3.4"
 132
 133 # Salt's dir
 134 ## states
 135 S_DIR="/srv/salt"
 136 ## formulas
 137 F_DIR="/srv/formulas"
 138 ##pillars
 139 P_DIR="/srv/pillars"
 140
 141 arguments ${@}
 142
 143 if [ -s ${CONFIG_FILE} ]; then
 144   source ${CONFIG_FILE}
 145 else
 146   echo >&2 "Please create a '${CONFIG_FILE}' file with initial values, as described in FIXME_URL_TO_DESCR"
 147   exit 1
 148 fi
 149
 150 if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
 151   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
 152   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
 153   exit 1
 154 fi
 155
 156 apt-get update
 157 apt-get install -y curl git jq
 158
 159 if which salt-call; then
 160   echo "Salt already installed"
 161 else
 162   curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 163   sh /tmp/bootstrap_salt.sh -XdfP -x python3
 164   /bin/systemctl stop salt-minion.service
 165   /bin/systemctl disable salt-minion.service
 166 fi
 167
 168 # Set salt to masterless mode
 169 cat > /etc/salt/minion << EOFSM
 170 file_client: local
 171 file_roots:
 172   base:
 173     - ${S_DIR}
 174     - ${F_DIR}/*
 175
 176 pillar_roots:
 177   base:
 178     - ${P_DIR}
 179 EOFSM
 180
 181 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
 182
 183 # Get the formula and dependencies
 184 cd ${F_DIR} || exit 1
 185 git clone --branch "${ARVADOS_TAG}" https://github.com/arvados/arvados-formula.git
 186 git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
 187 git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
 188 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 189 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 190
 191 if [ "x${BRANCH}" != "x" ]; then
 192   cd ${F_DIR}/arvados-formula || exit 1
 193   git checkout -t origin/"${BRANCH}"
 194   cd -
 195 fi
 196
 197 if [ "x${VAGRANT}" = "xyes" ]; then
 198   SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}/pillars"
 199   TESTS_DIR="/vagrant/${TESTS_DIR}"
 200 else
 201   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
 202   TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 203 fi
 204
 205 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
 206
 207 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
 208 # to ease deployment for newcomers
 209 for f in "${SOURCE_PILLARS_DIR}"/*; do
 210   sed "s/__ANONYMOUS_USER_TOKEN__/${ANONYMOUS_USER_TOKEN}/g;
 211        s/__BLOB_SIGNING_KEY__/${BLOB_SIGNING_KEY}/g;
 212        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 213        s/__CLUSTER__/${CLUSTER}/g;
 214        s/__DOMAIN__/${DOMAIN}/g;
 215        s/__HOSTNAME_EXT__/${HOSTNAME_EXT}/g;
 216        s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 217        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 218        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 219        s/__INITIAL_USER__/${INITIAL_USER}/g;
 220        s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 221        s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 222        s/__MANAGEMENT_TOKEN__/${MANAGEMENT_TOKEN}/g;
 223        s/__RELEASE__/${RELEASE}/g;
 224        s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g;
 225        s/__VERSION__/${VERSION}/g;
 226        s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 227        s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 228        s/__WORKBENCH1_EXT_SSL_PORT__/${WORKBENCH1_EXT_SSL_PORT}/g;
 229        s/__WORKBENCH2_EXT_SSL_PORT__/${WORKBENCH2_EXT_SSL_PORT}/g;
 230        s/__WORKBENCH_SECRET_KEY__/${WORKBENCH_SECRET_KEY}/g" \
 231   "${f}" > "${P_DIR}"/$(basename "${f}")
 232 done
 233
 234 mkdir -p /tmp/cluster_tests
 235 # Replace cluster and domain name in the test files
 236 for f in "${TESTS_DIR}"/*; do
 237   sed "s/__CLUSTER__/${CLUSTER}/g;
 238        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 239        s/__DOMAIN__/${DOMAIN}/g;
 240        s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 241        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 242        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g
 243        s/__INITIAL_USER__/${INITIAL_USER}/g;
 244        s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g" \
 245   "${f}" > "/tmp/cluster_tests"/$(basename "${f}")
 246 done
 247 chmod 755 /tmp/cluster_tests/run-test.sh
 248
 249 # Replace helper state files that differ from the formula's examples
 250 if [ -d "${SOURCE_STATES_DIR}" ]; then
 251   mkdir -p "${F_DIR}"/extra/extra
 252
 253   for f in "${SOURCE_STATES_DIR}"/*; do
 254     sed "s/__ANONYMOUS_USER_TOKEN__/${ANONYMOUS_USER_TOKEN}/g;
 255          s/__CLUSTER__/${CLUSTER}/g;
 256          s/__BLOB_SIGNING_KEY__/${BLOB_SIGNING_KEY}/g;
 257          s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 258          s/__DOMAIN__/${DOMAIN}/g;
 259          s/__HOSTNAME_EXT__/${HOSTNAME_EXT}/g;
 260          s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 261          s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 262          s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 263          s/__INITIAL_USER__/${INITIAL_USER}/g;
 264          s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 265          s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 266          s/__MANAGEMENT_TOKEN__/${MANAGEMENT_TOKEN}/g;
 267          s/__RELEASE__/${RELEASE}/g;
 268          s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g;
 269          s/__VERSION__/${VERSION}/g;
 270          s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 271          s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 272          s/__WORKBENCH1_EXT_SSL_PORT__/${WORKBENCH1_EXT_SSL_PORT}/g;
 273          s/__WORKBENCH2_EXT_SSL_PORT__/${WORKBENCH2_EXT_SSL_PORT}/g;
 274          s/__WORKBENCH_SECRET_KEY__/${WORKBENCH_SECRET_KEY}/g" \
 275     "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
 276   done
 277 fi
 278
 279 # Now, we build the SALT states/pillars trees
 280 # As we need to separate both states and pillars in case we want specific
 281 # roles, we iterate on both at the same time
 282
 283 # States
 284 cat > ${S_DIR}/top.sls << EOFTSLS
 285 base:
 286   '*':
 287     - locale
 288 EOFTSLS
 289
 290 # Pillars
 291 cat > ${P_DIR}/top.sls << EOFPSLS
 292 base:
 293   '*':
 294     - locale
 295     - arvados
 296 EOFPSLS
 297
 298 # States, extra states
 299 if [ -d "${F_DIR}"/extra/extra ]; then
 300   for f in "${F_DIR}"/extra/extra/*.sls; do
 301   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
 302   done
 303 fi
 304
 305 # If we want specific roles for a node, just add the desired states
 306 # and its dependencies
 307 if [ -z "${ROLES}" ]; then
 308   # States
 309   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 310   echo "    - postgres" >> ${S_DIR}/top.sls
 311   echo "    - docker" >> ${S_DIR}/top.sls
 312   echo "    - arvados" >> ${S_DIR}/top.sls
 313
 314   # Pillars
 315   echo "    - docker" >> ${P_DIR}/top.sls
 316   echo "    - nginx_api_configuration" >> ${P_DIR}/top.sls
 317   echo "    - nginx_controller_configuration" >> ${P_DIR}/top.sls
 318   echo "    - nginx_keepproxy_configuration" >> ${P_DIR}/top.sls
 319   echo "    - nginx_keepweb_configuration" >> ${P_DIR}/top.sls
 320   echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 321   echo "    - nginx_websocket_configuration" >> ${P_DIR}/top.sls
 322   echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 323   echo "    - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
 324   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
 325   echo "    - postgresql" >> ${P_DIR}/top.sls
 326 else
 327   # If we add individual roles, make sure we add the repo first
 328   echo "    - arvados.repo" >> ${S_DIR}/top.sls
 329   for R in ${ROLES}; do
 330     case "${R}" in
 331       "database")
 332         # States
 333         echo "    - postgres" >> ${S_DIR}/top.sls
 334         # Pillars
 335         echo '    - postgresql' >> ${P_DIR}/top.sls
 336       ;;
 337       "api")
 338         # States
 339         # FIXME: https://dev.arvados.org/issues/17352
 340         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
 341         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 342         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 343         # Pillars
 344         grep -q "docker" ${P_DIR}/top.sls                   || echo "    - docker" >> ${P_DIR}/top.sls
 345         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
 346         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 347         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 348       ;;
 349       "controller" | "websocket" | "workbench" | "workbench2" | "keepweb" | "keepproxy")
 350         # States
 351         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 352         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 353         # Pillars
 354         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 355         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 356       ;;
 357       "shell")
 358         # States
 359         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker" >> ${S_DIR}/top.sls
 360         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 361         # Pillars
 362         grep -q "" ${P_DIR}/top.sls                             || echo "    - docker" >> ${P_DIR}/top.sls
 363         grep -q "nginx_webshell_configuration" ${P_DIR}/top.sls || echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 364       ;;
 365       "dispatcher")
 366         # States
 367         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker" >> ${S_DIR}/top.sls
 368         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 369         # Pillars
 370         # ATM, no specific pillar needed
 371       ;;
 372       "keepstore")
 373         # States
 374         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 375         # Pillars
 376         # ATM, no specific pillar needed
 377       ;;
 378       *)
 379         echo "Unknown role ${R}"
 380         exit 1
 381       ;;
 382     esac
 383   done
 384 fi
 385
 386 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 387 if [ -e /root/.psqlrc ]; then
 388   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 389     RESTORE_PSQL="yes"
 390     cp /root/.psqlrc /root/.psqlrc.provision.backup
 391   fi
 392 else
 393   DELETE_PSQL="yes"
 394 fi
 395
 396 echo '\pset pager off' >> /root/.psqlrc
 397 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 398
 399 # Now run the install
 400 salt-call --local state.apply -l ${LOG_LEVEL}
 401
 402 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 403 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 404   echo "Removing .psql file"
 405   rm /root/.psqlrc
 406 fi
 407
 408 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 409   echo "Restoring .psql file"
 410   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 411 fi
 412 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 413
 414 # Leave a copy of the Arvados CA so the user can copy it where it's required
 415 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 416 # If running in a vagrant VM, also add default user to docker group
 417 if [ "x${VAGRANT}" = "xyes" ]; then
 418   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 419
 420   echo "Adding the vagrant user to the docker group"
 421   usermod -a -G docker vagrant
 422 else
 423   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 424 fi
 425
 426 # Test that the installation finished correctly
 427 if [ "x${TEST}" = "xyes" ]; then
 428   cd /tmp/cluster_tests
 429   ./run-test.sh
 430 fi