tools/salt-install/provision.sh

   1 #!/bin/bash -x
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14
  15 # capture the directory that the script is running from
  16 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  17
  18 usage() {
  19   echo >&2
  20   echo >&2 "Usage: ${0} [-h] [-h]"
  21   echo >&2
  22   echo >&2 "${0} options:"
  23   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  24   echo >&2 "  -p <N>, --ssl-port <N>                      SSL port to use for the web applications"
  25   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  26   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  27   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  28   echo >&2 "                                              Possible values are:"
  29   echo >&2 "                                                api"
  30   echo >&2 "                                                controller"
  31   echo >&2 "                                                keepstore"
  32   echo >&2 "                                                websocket"
  33   echo >&2 "                                                keepweb"
  34   echo >&2 "                                                workbench2"
  35   echo >&2 "                                                keepproxy"
  36   echo >&2 "                                                shell"
  37   echo >&2 "                                                workbench"
  38   echo >&2 "                                                dispatcher"
  39   echo >&2 "                                              Defaults to applying them all"
  40   echo >&2 "  -h, --help                                  Display this help and exit"
  41   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  42   echo >&2
  43 }
  44
  45 arguments() {
  46   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  47   TEMP=$(getopt -o c:dhp:r:tv \
  48     --long config:,debug,help,ssl-port:,roles:,test,vagrant \
  49     -n "${0}" -- "${@}")
  50
  51   if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
  52   # Note the quotes around `$TEMP': they are essential!
  53   eval set -- "$TEMP"
  54
  55   while [ ${#} -ge 1 ]; do
  56     case ${1} in
  57       -c | --config)
  58         CONFIG_FILE=${2}
  59         shift 2
  60         ;;
  61       -d | --debug)
  62         LOG_LEVEL="debug"
  63         shift
  64         ;;
  65       -p | --ssl-port)
  66         CONTROLLER_EXT_SSL_PORT=${2}
  67         shift 2
  68         ;;
  69       -r | --roles)
  70         for i in ${2//,/ }
  71           do
  72             # Verify the role exists
  73             if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
  74               echo "The role '${i}' is not a valid role"
  75               usage
  76               exit 1
  77             fi
  78             ROLES="${ROLES} ${i}"
  79           done
  80           shift 2
  81         ;;
  82       -t | --test)
  83         TEST="yes"
  84         shift
  85         ;;
  86       -v | --vagrant)
  87         VAGRANT="yes"
  88         shift
  89         ;;
  90       --)
  91         shift
  92         break
  93         ;;
  94       *)
  95         usage
  96         exit 1
  97         ;;
  98     esac
  99   done
 100 }
 101
 102 CONFIG="${SCRIPT_DIR}/local.params"
 103 CONFIG_DIR="config_examples/single_host/multiple_hostnames"
 104 LOG_LEVEL="info"
 105 CONTROLLER_EXT_SSL_PORT=443
 106 TESTS_DIR="tests"
 107
 108 CLUSTER=""
 109 DOMAIN=""
 110
 111 # Hostnames/IPs used for single-host deploys
 112 HOSTNAME_EXT=""
 113 HOSTNAME_INT="127.0.1.1"
 114
 115 # Initial user setup
 116 INITIAL_USER=""
 117 INITIAL_USER_EMAIL=""
 118 INITIAL_USER_PASSWORD=""
 119
 120 CONTROLLER_EXT_SSL_PORT=8000
 121 KEEP_EXT_SSL_PORT=25101
 122 # Both for collections and downloads
 123 KEEPWEB_EXT_SSL_PORT=9002
 124 WEBSHELL_EXT_SSL_PORT=4202
 125 WEBSOCKET_EXT_SSL_PORT=8002
 126 WORKBENCH1_EXT_SSL_PORT=443
 127 WORKBENCH2_EXT_SSL_PORT=3001
 128
 129 RELEASE="production"
 130 VERSION="latest"
 131 ARVADOS_TAG="v1.1.4"
 132 POSTGRES_TAG="v0.41.3"
 133 NGINX_TAG="v2.4.0"
 134 DOCKER_TAG="v1.0.0"
 135 LOCALE_TAG="v0.3.4"
 136
 137 # Salt's dir
 138 ## states
 139 S_DIR="/srv/salt"
 140 ## formulas
 141 F_DIR="/srv/formulas"
 142 ##pillars
 143 P_DIR="/srv/pillars"
 144
 145 arguments ${@}
 146
 147 if [ -s ${CONFIG_FILE} ]; then
 148   source ${CONFIG_FILE}
 149 else
 150   echo >&2 "Please create a '${CONFIG_FILE}' file with initial values, as described in FIXME_URL_TO_DESCR"
 151   exit 1
 152 fi
 153
 154 if grep -q 'fixme_or_this_wont_work' ${CONFIG_FILE} ; then
 155   echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified."
 156   echo >&2 "Please, fix them and re-run the provision script."
 157   exit 1
 158 fi
 159
 160 if ! grep -E '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
 161   echo >&2 "ERROR: <CLUSTER> must be exactly 5 alphanumeric characters long"
 162   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
 163   exit 1
 164 fi
 165
 166 apt-get update
 167 apt-get install -y curl git jq
 168
 169 if which salt-call; then
 170   echo "Salt already installed"
 171 else
 172   curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 173   sh /tmp/bootstrap_salt.sh -XdfP -x python3
 174   /bin/systemctl stop salt-minion.service
 175   /bin/systemctl disable salt-minion.service
 176 fi
 177
 178 # Set salt to masterless mode
 179 cat > /etc/salt/minion << EOFSM
 180 file_client: local
 181 file_roots:
 182   base:
 183     - ${S_DIR}
 184     - ${F_DIR}/*
 185
 186 pillar_roots:
 187   base:
 188     - ${P_DIR}
 189 EOFSM
 190
 191 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
 192
 193 # Get the formula and dependencies
 194 cd ${F_DIR} || exit 1
 195 git clone --branch "${ARVADOS_TAG}" https://github.com/arvados/arvados-formula.git
 196 git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
 197 git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
 198 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 199 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 200
 201 # If we want to try a specific branch of the formula
 202 if [ "x${BRANCH}" != "x" ]; then
 203   cd ${F_DIR}/arvados-formula || exit 1
 204   git checkout -t origin/"${BRANCH}" -b "${BRANCH}"
 205   cd -
 206 fi
 207
 208 if [ "x${VAGRANT}" = "xyes" ]; then
 209   SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}/pillars"
 210   SOURCE_TESTS_DIR="/vagrant/${TESTS_DIR}"
 211 else
 212   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
 213   SOURCE_TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 214 fi
 215
 216 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
 217
 218 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
 219 # to ease deployment for newcomers
 220 for f in "${SOURCE_PILLARS_DIR}"/*; do
 221   sed "s/__ANONYMOUS_USER_TOKEN__/${ANONYMOUS_USER_TOKEN}/g;
 222        s/__BLOB_SIGNING_KEY__/${BLOB_SIGNING_KEY}/g;
 223        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 224        s/__CLUSTER__/${CLUSTER}/g;
 225        s/__DOMAIN__/${DOMAIN}/g;
 226        s/__HOSTNAME_EXT__/${HOSTNAME_EXT}/g;
 227        s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 228        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 229        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 230        s/__INITIAL_USER__/${INITIAL_USER}/g;
 231        s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 232        s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 233        s/__MANAGEMENT_TOKEN__/${MANAGEMENT_TOKEN}/g;
 234        s/__RELEASE__/${RELEASE}/g;
 235        s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g;
 236        s/__VERSION__/${VERSION}/g;
 237        s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 238        s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 239        s/__WORKBENCH1_EXT_SSL_PORT__/${WORKBENCH1_EXT_SSL_PORT}/g;
 240        s/__WORKBENCH2_EXT_SSL_PORT__/${WORKBENCH2_EXT_SSL_PORT}/g;
 241        s/__WORKBENCH_SECRET_KEY__/${WORKBENCH_SECRET_KEY}/g" \
 242   "${f}" > "${P_DIR}"/$(basename "${f}")
 243 done
 244
 245 mkdir -p /tmp/cluster_tests
 246 # Replace cluster and domain name in the test files
 247 for f in "${SOURCE_TESTS_DIR}"/*; do
 248   sed "s/__CLUSTER__/${CLUSTER}/g;
 249        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 250        s/__DOMAIN__/${DOMAIN}/g;
 251        s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 252        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 253        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g
 254        s/__INITIAL_USER__/${INITIAL_USER}/g;
 255        s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g" \
 256   "${f}" > "/tmp/cluster_tests"/$(basename "${f}")
 257 done
 258 chmod 755 /tmp/cluster_tests/run-test.sh
 259
 260 # Replace helper state files that differ from the formula's examples
 261 if [ -d "${SOURCE_STATES_DIR}" ]; then
 262   mkdir -p "${F_DIR}"/extra/extra
 263
 264   for f in "${SOURCE_STATES_DIR}"/*; do
 265     sed "s/__ANONYMOUS_USER_TOKEN__/${ANONYMOUS_USER_TOKEN}/g;
 266          s/__CLUSTER__/${CLUSTER}/g;
 267          s/__BLOB_SIGNING_KEY__/${BLOB_SIGNING_KEY}/g;
 268          s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 269          s/__DOMAIN__/${DOMAIN}/g;
 270          s/__HOSTNAME_EXT__/${HOSTNAME_EXT}/g;
 271          s/__HOSTNAME_INT__/${HOSTNAME_INT}/g;
 272          s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 273          s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 274          s/__INITIAL_USER__/${INITIAL_USER}/g;
 275          s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 276          s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 277          s/__MANAGEMENT_TOKEN__/${MANAGEMENT_TOKEN}/g;
 278          s/__RELEASE__/${RELEASE}/g;
 279          s/__SYSTEM_ROOT_TOKEN__/${SYSTEM_ROOT_TOKEN}/g;
 280          s/__VERSION__/${VERSION}/g;
 281          s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 282          s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 283          s/__WORKBENCH1_EXT_SSL_PORT__/${WORKBENCH1_EXT_SSL_PORT}/g;
 284          s/__WORKBENCH2_EXT_SSL_PORT__/${WORKBENCH2_EXT_SSL_PORT}/g;
 285          s/__WORKBENCH_SECRET_KEY__/${WORKBENCH_SECRET_KEY}/g" \
 286     "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
 287   done
 288 fi
 289
 290 # Now, we build the SALT states/pillars trees
 291 # As we need to separate both states and pillars in case we want specific
 292 # roles, we iterate on both at the same time
 293
 294 # States
 295 cat > ${S_DIR}/top.sls << EOFTSLS
 296 base:
 297   '*':
 298     - locale
 299 EOFTSLS
 300
 301 # Pillars
 302 cat > ${P_DIR}/top.sls << EOFPSLS
 303 base:
 304   '*':
 305     - locale
 306     - arvados
 307 EOFPSLS
 308
 309 # States, extra states
 310 if [ -d "${F_DIR}"/extra/extra ]; then
 311   for f in "${F_DIR}"/extra/extra/*.sls; do
 312   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
 313   done
 314 fi
 315
 316 # If we want specific roles for a node, just add the desired states
 317 # and its dependencies
 318 if [ -z "${ROLES}" ]; then
 319   # States
 320   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 321   echo "    - postgres" >> ${S_DIR}/top.sls
 322   echo "    - docker" >> ${S_DIR}/top.sls
 323   echo "    - arvados" >> ${S_DIR}/top.sls
 324
 325   # Pillars
 326   echo "    - docker" >> ${P_DIR}/top.sls
 327   echo "    - nginx_api_configuration" >> ${P_DIR}/top.sls
 328   echo "    - nginx_controller_configuration" >> ${P_DIR}/top.sls
 329   echo "    - nginx_keepproxy_configuration" >> ${P_DIR}/top.sls
 330   echo "    - nginx_keepweb_configuration" >> ${P_DIR}/top.sls
 331   echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 332   echo "    - nginx_websocket_configuration" >> ${P_DIR}/top.sls
 333   echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 334   echo "    - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
 335   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
 336   echo "    - postgresql" >> ${P_DIR}/top.sls
 337 else
 338   # If we add individual roles, make sure we add the repo first
 339   echo "    - arvados.repo" >> ${S_DIR}/top.sls
 340   for R in ${ROLES}; do
 341     case "${R}" in
 342       "database")
 343         # States
 344         echo "    - postgres" >> ${S_DIR}/top.sls
 345         # Pillars
 346         echo '    - postgresql' >> ${P_DIR}/top.sls
 347       ;;
 348       "api")
 349         # States
 350         # FIXME: https://dev.arvados.org/issues/17352
 351         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
 352         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 353         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 354         # Pillars
 355         grep -q "docker" ${P_DIR}/top.sls                   || echo "    - docker" >> ${P_DIR}/top.sls
 356         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
 357         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 358         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 359       ;;
 360       "controller" | "websocket" | "workbench" | "workbench2" | "keepweb" | "keepproxy")
 361         # States
 362         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
 363         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 364         # Pillars
 365         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
 366         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
 367       ;;
 368       "shell")
 369         # States
 370         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker" >> ${S_DIR}/top.sls
 371         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 372         # Pillars
 373         grep -q "" ${P_DIR}/top.sls                             || echo "    - docker" >> ${P_DIR}/top.sls
 374         grep -q "nginx_webshell_configuration" ${P_DIR}/top.sls || echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
 375       ;;
 376       "dispatcher")
 377         # States
 378         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker" >> ${S_DIR}/top.sls
 379         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 380         # Pillars
 381         # ATM, no specific pillar needed
 382       ;;
 383       "keepstore")
 384         # States
 385         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 386         # Pillars
 387         # ATM, no specific pillar needed
 388       ;;
 389       *)
 390         echo "Unknown role ${R}"
 391         exit 1
 392       ;;
 393     esac
 394   done
 395 fi
 396
 397 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 398 if [ -e /root/.psqlrc ]; then
 399   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 400     RESTORE_PSQL="yes"
 401     cp /root/.psqlrc /root/.psqlrc.provision.backup
 402   fi
 403 else
 404   DELETE_PSQL="yes"
 405 fi
 406
 407 echo '\pset pager off' >> /root/.psqlrc
 408 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 409
 410 # Now run the install
 411 salt-call --local state.apply -l ${LOG_LEVEL}
 412
 413 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 414 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 415   echo "Removing .psql file"
 416   rm /root/.psqlrc
 417 fi
 418
 419 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 420   echo "Restoring .psql file"
 421   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 422 fi
 423 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 424
 425 # Leave a copy of the Arvados CA so the user can copy it where it's required
 426 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 427 # If running in a vagrant VM, also add default user to docker group
 428 if [ "x${VAGRANT}" = "xyes" ]; then
 429   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 430
 431   echo "Adding the vagrant user to the docker group"
 432   usermod -a -G docker vagrant
 433 else
 434   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
 435 fi
 436
 437 # Test that the installation finished correctly
 438 if [ "x${TEST}" = "xyes" ]; then
 439   cd /tmp/cluster_tests
 440   ./run-test.sh
 441 fi