tools/salt-install/provision.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14
  15 # capture the directory that the script is running from
  16 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  17
  18 usage() {
  19   echo >&2
  20   echo >&2 "Usage: ${0} [-h] [-h]"
  21   echo >&2
  22   echo >&2 "${0} options:"
  23   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  24   echo >&2 "  -p <N>, --ssl-port <N>                      SSL port to use for the web applications"
  25   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  26   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  27   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  28   echo >&2 "                                              Possible values are:"
  29   echo >&2 "                                                api"
  30   echo >&2 "                                                controller"
  31   echo >&2 "                                                keepstore"
  32   echo >&2 "                                                websocket"
  33   echo >&2 "                                                keepweb"
  34   echo >&2 "                                                workbench2"
  35   echo >&2 "                                                keepproxy"
  36   echo >&2 "                                                shell"
  37   echo >&2 "                                                workbench"
  38   echo >&2 "                                                dispatcher"
  39   echo >&2 "                                              Defaults to applying them all"
  40   echo >&2 "  -h, --help                                  Display this help and exit"
  41   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  42   echo >&2
  43 }
  44
  45 arguments() {
  46   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  47   TEMP=$(getopt -o c:dhp:r:tv \
  48     --long config:,debug,help,ssl-port:,roles:,test,vagrant \
  49     -n "${0}" -- "${@}")
  50
  51   if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
  52   # Note the quotes around `$TEMP': they are essential!
  53   eval set -- "$TEMP"
  54
  55   while [ ${#} -ge 1 ]; do
  56     case ${1} in
  57       -c | --config)
  58         CONFIG=${2}
  59         shift 2
  60         ;;
  61       -d | --debug)
  62         LOG_LEVEL="debug"
  63         shift
  64         ;;
  65       -p | --ssl-port)
  66         HOST_SSL_PORT=${2}
  67         shift 2
  68         ;;
  69       -r | --roles)
  70         for i in ${2//,/ }
  71           do
  72             # Verify the role exists
  73             if [[ ! "api,controller,keepstore,websocket,keepweb,workbench2,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
  74               echo "The role '${i}' is not a valid role"
  75               usage
  76               exit 1
  77             fi
  78             ROLES="${ROLES} ${i}"
  79           done
  80           shift 2
  81         ;;
  82       -t | --test)
  83         TEST="yes"
  84         shift
  85         ;;
  86       -v | --vagrant)
  87         VAGRANT="yes"
  88         shift
  89         ;;
  90       --)
  91         shift
  92         break
  93         ;;
  94       *)
  95         usage
  96         exit 1
  97         ;;
  98     esac
  99   done
 100 }
 101
 102 CONFIG="${SCRIPT_DIR}/local.params"
 103 CONFIG_DIR="config_examples/single_host/multiple_hostnames"
 104 LOG_LEVEL="info"
 105 HOST_SSL_PORT=443
 106 TESTS_DIR="tests"
 107
 108 CLUSTER=""
 109 DOMAIN=""
 110 HOSTNAME=""
 111 INITIAL_USER=""
 112 INITIAL_USER_EMAIL=""
 113 INITIAL_USER_PASSWORD=""
 114
 115 CONTROLLER_EXT_SSL_PORT=8000
 116 KEEP_EXT_SSL_PORT=25101
 117 # Both for collections and downloads
 118 KEEPWEB_EXT_SSL_PORT=9002
 119 WEBSHELL_EXT_SSL_PORT=4202
 120 WEBSOCKET_EXT_SSL_PORT=8002
 121 WORKBENCH1_EXT_SSL_PORT=443
 122 WORKBENCH2_EXT_SSL_PORT=3001
 123
 124 RELEASE="production"
 125 VERSION="latest"
 126 ARVADOS_TAG="v1.1.4"
 127 POSTGRES_TAG="v0.41.3"
 128 NGINX_TAG="v2.4.0"
 129 DOCKER_TAG="v1.0.0"
 130 LOCALE_TAG="v0.3.4"
 131
 132 arguments ${@}
 133
 134 if [ -s ${CONFIG} ]; then
 135   source ${CONFIG}
 136 else
 137   echo >&2 "Please create a '${CONFIG}' file with initial values, as described in FIXME_URL_TO_DESCR"
 138   exit 1
 139 fi
 140
 141 # Salt's dir
 142 ## states
 143 S_DIR="/srv/salt"
 144 ## formulas
 145 F_DIR="/srv/formulas"
 146 ##pillars
 147 P_DIR="/srv/pillars"
 148
 149 apt-get update
 150 apt-get install -y curl git jq
 151
 152 dpkg -l |grep salt-minion
 153 if [ ${?} -eq 0 ]; then
 154   echo "Salt already installed"
 155 else
 156   curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 157   sh /tmp/bootstrap_salt.sh -XUdfP -x python3
 158   /bin/systemctl disable salt-minion.service
 159 fi
 160
 161 # Set salt to masterless mode
 162 cat > /etc/salt/minion << EOFSM
 163 file_client: local
 164 file_roots:
 165   base:
 166     - ${S_DIR}
 167     - ${F_DIR}/*
 168     - ${F_DIR}/*/test/salt/states/examples
 169
 170 pillar_roots:
 171   base:
 172     - ${P_DIR}
 173 EOFSM
 174
 175 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
 176
 177 # States
 178 cat > ${S_DIR}/top.sls << EOFTSLS
 179 base:
 180   '*':
 181     - single_host.host_entries
 182     - single_host.snakeoil_certs
 183     - locale
 184     - nginx.passenger
 185     - postgres
 186     - docker
 187 EOFTSLS
 188
 189 # If we want specific roles for a node, just add those states
 190 if [ -z "${ROLES}" ]; then
 191   echo '    - arvados' >> ${S_DIR}/top.sls
 192 else
 193   for R in ${ROLES}; do
 194     echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 195   done
 196 fi
 197
 198 # Pillars
 199 cat > ${P_DIR}/top.sls << EOFPSLS
 200 base:
 201   '*':
 202     - arvados
 203     - docker
 204     - locale
 205     - nginx_api_configuration
 206     - nginx_controller_configuration
 207     - nginx_keepproxy_configuration
 208     - nginx_keepweb_configuration
 209     - nginx_passenger
 210     - nginx_websocket_configuration
 211     - nginx_webshell_configuration
 212     - nginx_workbench2_configuration
 213     - nginx_workbench_configuration
 214     - postgresql
 215 EOFPSLS
 216
 217 # Get the formula and dependencies
 218 cd ${F_DIR} || exit 1
 219 git clone --branch "${ARVADOS_TAG}" https://github.com/arvados/arvados-formula.git
 220 git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
 221 git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
 222 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 223 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 224
 225 if [ "x${BRANCH}" != "x" ]; then
 226   cd ${F_DIR}/arvados-formula || exit 1
 227   git checkout -t origin/"${BRANCH}"
 228   cd -
 229 fi
 230
 231 if [ "x${VAGRANT}" = "xyes" ]; then
 232   SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
 233   TESTS_DIR="/vagrant/${TESTS_DIR}"
 234 else
 235   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}"
 236   TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 237 fi
 238
 239 # Replace cluster and domain name in the example pillars and test files
 240 for f in "${SOURCE_PILLARS_DIR}"/*; do
 241   sed "s/__CLUSTER__/${CLUSTER}/g;
 242        s/__DOMAIN__/${DOMAIN}/g;
 243        s/__RELEASE__/${RELEASE}/g;
 244        s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
 245        s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
 246        s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
 247        s/__WORKBENCH1_EXT__SSL_PORT__/${WORKBENCH1_EXT__SSL_PORT}/g;
 248        s/__WORKBENCH2_EXT__SSL_PORT__/${WORKBENCH2_EXT__SSL_PORT}/g;
 249        s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
 250        s/__HOSTNAME__/${HOSTNAME}/g;
 251        s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
 252        s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
 253        s/__INITIAL_USER__/${INITIAL_USER}/g;
 254        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 255        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 256        s/__VERSION__/${VERSION}/g" \
 257   "${f}" > "${P_DIR}"/$(basename "${f}")
 258 done
 259
 260 mkdir -p /tmp/cluster_tests
 261 # Replace cluster and domain name in the example pillars and test files
 262 for f in "${TESTS_DIR}"/*; do
 263   sed "s/__CLUSTER__/${CLUSTER}/g;
 264        s/__DOMAIN__/${DOMAIN}/g;
 265        s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
 266        s/__INITIAL_USER__/${INITIAL_USER}/g;
 267        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 268        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \
 269   ${f} > /tmp/cluster_tests/$(basename ${f})
 270 done
 271 chmod 755 /tmp/cluster_tests/run-test.sh
 272
 273 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 274 if [ -e /root/.psqlrc ]; then
 275   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 276     RESTORE_PSQL="yes"
 277     cp /root/.psqlrc /root/.psqlrc.provision.backup
 278   fi
 279 else
 280   DELETE_PSQL="yes"
 281 fi
 282
 283 echo '\pset pager off' >> /root/.psqlrc
 284 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 285
 286 # Now run the install
 287 salt-call --local state.apply -l ${LOG_LEVEL}
 288
 289 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 290 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 291   echo "Removing .psql file"
 292   rm /root/.psqlrc
 293 fi
 294
 295 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 296   echo "Restoring .psql file"
 297   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 298 fi
 299 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 300
 301 # Leave a copy of the Arvados CA so the user can copy it where it's required
 302 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 303 # If running in a vagrant VM, also add default user to docker group
 304 if [ "x${VAGRANT}" = "xyes" ]; then
 305   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant
 306
 307   echo "Adding the vagrant user to the docker group"
 308   usermod -a -G docker vagrant
 309 else
 310   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}
 311 fi
 312
 313 # Test that the installation finished correctly
 314 if [ "x${TEST}" = "xyes" ]; then
 315   cd /tmp/cluster_tests
 316   ./run-test.sh
 317 fi