tools/salt-install/provision.sh

   1 #!/bin/bash
   2
   3 # Copyright (C) The Arvados Authors. All rights reserved.
   4 #
   5 # SPDX-License-Identifier: CC-BY-SA-3.0
   6
   7 # If you want to test arvados in a single host, you can run this script, which
   8 # will install it using salt masterless
   9 # This script is run by the Vagrant file when you run it with
  10 #
  11 # vagrant up
  12
  13 set -o pipefail
  14
  15 # capture the directory that the script is running from
  16 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  17
  18 CONFIG_DIR="single_host"
  19 RELEASE="production"
  20 VERSION="latest"
  21 ARVADOS_TAG="v1.1.4"
  22 POSTGRES_TAG="v0.41.3"
  23 NGINX_TAG="v2.4.0"
  24 DOCKER_TAG="v1.0.0"
  25 LOCALE_TAG="v0.3.4"
  26
  27 usage() {
  28   echo >&2
  29   echo >&2 "Usage: ${0} [-h] [-h]"
  30   echo >&2
  31   echo >&2 "${0} options:"
  32   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
  33   echo >&2 "  -p <N>, --ssl-port <N>                      SSL port to use for the web applications"
  34   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
  35   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
  36   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
  37   echo >&2 "                                              Possible values are:"
  38   echo >&2 "                                                api"
  39   echo >&2 "                                                controller"
  40   echo >&2 "                                                keepstore"
  41   echo >&2 "                                                websocket"
  42   echo >&2 "                                                keepweb"
  43   echo >&2 "                                                workbench2"
  44   echo >&2 "                                                keepproxy"
  45   echo >&2 "                                                shell"
  46   echo >&2 "                                                workbench"
  47   echo >&2 "                                                dispatcher"
  48   echo >&2 "                                              Defaults to applying them all"
  49   echo >&2 "  -h, --help                                  Display this help and exit"
  50   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
  51   echo >&2
  52 }
  53
  54 arguments() {
  55   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
  56   TEMP=$(getopt -o c:dhp:r:tv \
  57     --long config:,debug,help,ssl-port:,roles:,test,vagrant \
  58     -n "${0}" -- "${@}")
  59
  60   if [ ${?} != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
  61   # Note the quotes around `$TEMP': they are essential!
  62   eval set -- "$TEMP"
  63
  64   while [ ${#} -ge 1 ]; do
  65     case ${1} in
  66       -c | --config)
  67         CONFIG=${2}
  68         shift 2
  69         ;;
  70       -d | --debug)
  71         LOG_LEVEL="debug"
  72         shift
  73         ;;
  74       -p | --ssl-port)
  75         HOST_SSL_PORT=${2}
  76         shift 2
  77         ;;
  78       -r | --roles)
  79         for i in ${2//,/ }
  80           do
  81             # Verify the role exists
  82             if [[ ! "api,controller,keepstore,websocket,keepweb,workbench2,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
  83               echo "The role '${i}' is not a valid role"
  84               usage
  85               exit 1
  86             fi
  87             ROLES="${ROLES} ${i}"
  88           done
  89           shift 2
  90         ;;
  91       -t | --test)
  92         TEST="yes"
  93         shift
  94         ;;
  95       -v | --vagrant)
  96         VAGRANT="yes"
  97         shift
  98         ;;
  99       --)
 100         shift
 101         break
 102         ;;
 103       *)
 104         usage
 105         exit 1
 106         ;;
 107     esac
 108   done
 109 }
 110
 111 CONFIG="${SCRIPT_DIR}/local.params"
 112 LOG_LEVEL="info"
 113 HOST_SSL_PORT=443
 114 TESTS_DIR="tests"
 115
 116 arguments ${@}
 117
 118 if [ -s ${CONFIG} ]; then
 119   source ${CONFIG}
 120 else
 121   echo >&2 "Please create a '${CONFIG}' file with initial values, as described in FIXME_URL_TO_DESCR"
 122   exit 1
 123 fi
 124
 125 # Salt's dir
 126 ## states
 127 S_DIR="/srv/salt"
 128 ## formulas
 129 F_DIR="/srv/formulas"
 130 ##pillars
 131 P_DIR="/srv/pillars"
 132
 133 apt-get update
 134 apt-get install -y curl git jq
 135
 136 dpkg -l |grep salt-minion
 137 if [ ${?} -eq 0 ]; then
 138   echo "Salt already installed"
 139 else
 140   curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 141   sh /tmp/bootstrap_salt.sh -XUdfP -x python3
 142   /bin/systemctl disable salt-minion.service
 143 fi
 144
 145 # Set salt to masterless mode
 146 cat > /etc/salt/minion << EOFSM
 147 file_client: local
 148 file_roots:
 149   base:
 150     - ${S_DIR}
 151     - ${F_DIR}/*
 152     - ${F_DIR}/*/test/salt/states/examples
 153
 154 pillar_roots:
 155   base:
 156     - ${P_DIR}
 157 EOFSM
 158
 159 mkdir -p ${S_DIR}
 160 mkdir -p ${F_DIR}
 161 mkdir -p ${P_DIR}
 162
 163 # States
 164 cat > ${S_DIR}/top.sls << EOFTSLS
 165 base:
 166   '*':
 167     - single_host.host_entries
 168     - single_host.snakeoil_certs
 169     - locale
 170     - nginx.passenger
 171     - postgres
 172     - docker
 173 EOFTSLS
 174
 175 # If we want specific roles for a node, just add those states
 176 if [ -z "${ROLES}" ]; then
 177   echo '    - arvados' >> ${S_DIR}/top.sls
 178 else
 179   for R in ${ROLES}; do
 180     echo "    - arvados.${R}" >> ${S_DIR}/top.sls
 181   done
 182 fi
 183
 184 # Pillars
 185 cat > ${P_DIR}/top.sls << EOFPSLS
 186 base:
 187   '*':
 188     - arvados
 189     - docker
 190     - locale
 191     - nginx_api_configuration
 192     - nginx_controller_configuration
 193     - nginx_keepproxy_configuration
 194     - nginx_keepweb_configuration
 195     - nginx_passenger
 196     - nginx_websocket_configuration
 197     - nginx_webshell_configuration
 198     - nginx_workbench2_configuration
 199     - nginx_workbench_configuration
 200     - postgresql
 201 EOFPSLS
 202
 203 # Get the formula and dependencies
 204 cd ${F_DIR} || exit 1
 205 git clone --branch "${ARVADOS_TAG}" https://github.com/arvados/arvados-formula.git
 206 git clone --branch "${DOCKER_TAG}" https://github.com/saltstack-formulas/docker-formula.git
 207 git clone --branch "${LOCALE_TAG}" https://github.com/saltstack-formulas/locale-formula.git
 208 git clone --branch "${NGINX_TAG}" https://github.com/saltstack-formulas/nginx-formula.git
 209 git clone --branch "${POSTGRES_TAG}" https://github.com/saltstack-formulas/postgres-formula.git
 210
 211 if [ "x${BRANCH}" != "x" ]; then
 212   cd ${F_DIR}/arvados-formula || exit 1
 213   git checkout -t origin/"${BRANCH}"
 214   cd -
 215 fi
 216
 217 if [ "x${VAGRANT}" = "xyes" ]; then
 218   SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
 219   TESTS_DIR="/vagrant/${TESTS_DIR}"
 220 else
 221   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}"
 222   TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
 223 fi
 224
 225 # Replace cluster and domain name in the example pillars and test files
 226 for f in "${SOURCE_PILLARS_DIR}"/*; do
 227   sed "s/__CLUSTER__/${CLUSTER}/g;
 228        s/__DOMAIN__/${DOMAIN}/g;
 229        s/__RELEASE__/${RELEASE}/g;
 230        s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
 231        s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
 232        s/__INITIAL_USER__/${INITIAL_USER}/g;
 233        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 234        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;
 235        s/__VERSION__/${VERSION}/g" \
 236   "${f}" > "${P_DIR}"/$(basename "${f}")
 237 done
 238
 239 mkdir -p /tmp/cluster_tests
 240 # Replace cluster and domain name in the example pillars and test files
 241 for f in "${TESTS_DIR}"/*; do
 242   sed "s/__CLUSTER__/${CLUSTER}/g;
 243        s/__DOMAIN__/${DOMAIN}/g;
 244        s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
 245        s/__INITIAL_USER__/${INITIAL_USER}/g;
 246        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
 247        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g" \
 248   ${f} > /tmp/cluster_tests/$(basename ${f})
 249 done
 250 chmod 755 /tmp/cluster_tests/run-test.sh
 251
 252 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 253 if [ -e /root/.psqlrc ]; then
 254   if ! ( grep 'pset pager off' /root/.psqlrc ); then
 255     RESTORE_PSQL="yes"
 256     cp /root/.psqlrc /root/.psqlrc.provision.backup
 257   fi
 258 else
 259   DELETE_PSQL="yes"
 260 fi
 261
 262 echo '\pset pager off' >> /root/.psqlrc
 263 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 264
 265 # Now run the install
 266 salt-call --local state.apply -l ${LOG_LEVEL}
 267
 268 # FIXME! #16992 Temporary fix for psql call in arvados-api-server
 269 if [ "x${DELETE_PSQL}" = "xyes" ]; then
 270   echo "Removing .psql file"
 271   rm /root/.psqlrc
 272 fi
 273
 274 if [ "x${RESTORE_PSQL}" = "xyes" ]; then
 275   echo "Restoring .psql file"
 276   mv -v /root/.psqlrc.provision.backup /root/.psqlrc
 277 fi
 278 # END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 279
 280 # Leave a copy of the Arvados CA so the user can copy it where it's required
 281 echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
 282 # If running in a vagrant VM, also add default user to docker group
 283 if [ "x${VAGRANT}" = "xyes" ]; then
 284   cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant
 285
 286   echo "Adding the vagrant user to the docker group"
 287   usermod -a -G docker vagrant
 288 else
 289   cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}
 290 fi
 291
 292 # Test that the installation finished correctly
 293 if [ "x${TEST}" = "xyes" ]; then
 294   cd /tmp/cluster_tests
 295   ./run-test.sh
 296 fi