18870: Check for 'fixme's
[arvados.git] / tools / salt-install / provision.sh
1 #!/bin/bash
2
3 # Copyright (C) The Arvados Authors. All rights reserved.
4 #
5 # SPDX-License-Identifier: CC-BY-SA-3.0
6
7 # If you want to test arvados in a single host, you can run this script, which
8 # will install it using salt masterless
9 # This script is run by the Vagrant file when you run it with
10 #
11 # vagrant up
12
13 set -o pipefail
14
15 _exit_handler() {
16   local rc="$?"
17   trap - EXIT
18   if [ "$rc" -ne 0 ]; then
19     echo "Error occurred ($rc) while running $0 at line $1 : $BASH_COMMAND"
20   fi
21   exit "$rc"
22 }
23
24 trap '_exit_handler $LINENO' EXIT ERR
25
26 # capture the directory that the script is running from
27 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
28
29 usage() {
30   echo >&2
31   echo >&2 "Usage: ${0} [-h] [-h]"
32   echo >&2
33   echo >&2 "${0} options:"
34   echo >&2 "  -d, --debug                                 Run salt installation in debug mode"
35   echo >&2 "  -c <local.params>, --config <local.params>  Path to the local.params config file"
36   echo >&2 "  -t, --test                                  Test installation running a CWL workflow"
37   echo >&2 "  -r, --roles                                 List of Arvados roles to apply to the host, comma separated"
38   echo >&2 "                                              Possible values are:"
39   echo >&2 "                                                api"
40   echo >&2 "                                                controller"
41   echo >&2 "                                                dispatcher"
42   echo >&2 "                                                keepproxy"
43   echo >&2 "                                                keepbalance"
44   echo >&2 "                                                keepstore"
45   echo >&2 "                                                keepweb"
46   echo >&2 "                                                shell"
47   echo >&2 "                                                webshell"
48   echo >&2 "                                                websocket"
49   echo >&2 "                                                workbench"
50   echo >&2 "                                                workbench2"
51   echo >&2 "                                              Defaults to applying them all"
52   echo >&2 "  -h, --help                                  Display this help and exit"
53   echo >&2 "  --dump-config <dest_dir>                    Dumps the pillars and states to a directory"
54   echo >&2 "                                              This parameter does not perform any installation at all. It's"
55   echo >&2 "                                              intended to give you a parsed set of configuration files so"
56   echo >&2 "                                              you can inspect them or use them in you Saltstack infrastructure."
57   echo >&2 "                                              It"
58   echo >&2 "                                                - parses the pillar and states templates,"
59   echo >&2 "                                                - downloads the helper formulas with their desired versions,"
60   echo >&2 "                                                - prepares the 'top.sls' files both for pillars and states"
61   echo >&2 "                                                  for the selected role(s)"
62   echo >&2 "                                                - writes the resulting files into <dest_dir>"
63   echo >&2 "  -v, --vagrant                               Run in vagrant and use the /vagrant shared dir"
64   echo >&2 "  --development                               Run in dev mode, using snakeoil certs"
65   echo >&2
66 }
67
68 arguments() {
69   # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
70   if ! which getopt > /dev/null; then
71     echo >&2 "GNU getopt is required to run this script. Please install it and re-reun it"
72     exit 1
73   fi
74
75   TEMP=$(getopt -o c:dhp:r:tv \
76     --long config:,debug,development,dump-config:,help,roles:,test,vagrant \
77     -n "${0}" -- "${@}")
78
79   if [ ${?} != 0 ];
80     then echo "Please check the parameters you entered and re-run again"
81     exit 1
82   fi
83   # Note the quotes around `$TEMP': they are essential!
84   eval set -- "$TEMP"
85
86   while [ ${#} -ge 1 ]; do
87     case ${1} in
88       -c | --config)
89         CONFIG_FILE=${2}
90         shift 2
91         ;;
92       -d | --debug)
93         LOG_LEVEL="debug"
94         shift
95         set -x
96         ;;
97       --dump-config)
98         if [[ ${2} = /* ]]; then
99           DUMP_SALT_CONFIG_DIR=${2}
100         else
101           DUMP_SALT_CONFIG_DIR=${PWD}/${2}
102         fi
103         ## states
104         S_DIR="${DUMP_SALT_CONFIG_DIR}/salt"
105         ## formulas
106         F_DIR="${DUMP_SALT_CONFIG_DIR}/formulas"
107         ## pillars
108         P_DIR="${DUMP_SALT_CONFIG_DIR}/pillars"
109         ## tests
110         T_DIR="${DUMP_SALT_CONFIG_DIR}/tests"
111         DUMP_CONFIG="yes"
112         shift 2
113         ;;
114       --development)
115         DEV_MODE="yes"
116         shift 1
117         ;;
118       -r | --roles)
119         for i in ${2//,/ }
120           do
121             # Verify the role exists
122             if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
123               echo "The role '${i}' is not a valid role"
124               usage
125               exit 1
126             fi
127             ROLES="${ROLES} ${i}"
128           done
129           shift 2
130         ;;
131       -t | --test)
132         TEST="yes"
133         shift
134         ;;
135       -v | --vagrant)
136         VAGRANT="yes"
137         shift
138         ;;
139       --)
140         shift
141         break
142         ;;
143       *)
144         usage
145         exit 1
146         ;;
147     esac
148   done
149 }
150
151 copy_custom_cert() {
152   cert_dir=${1}
153   cert_name=${2}
154
155   mkdir -p /srv/salt/certs
156
157   if [ -f ${cert_dir}/${cert_name}.crt ]; then
158     cp -v ${cert_dir}/${cert_name}.crt /srv/salt/certs/arvados-${cert_name}.pem
159   else
160     echo "${cert_dir}/${cert_name}.crt does not exist. Exiting"
161     exit 1
162   fi
163   if [ -f ${cert_dir}/${cert_name}.key ]; then
164     cp -v ${cert_dir}/${cert_name}.key /srv/salt/certs/arvados-${cert_name}.key
165   else
166     echo "${cert_dir}/${cert_name}.key does not exist. Exiting"
167     exit 1
168   fi
169 }
170
171 DEV_MODE="no"
172 CONFIG_FILE="${SCRIPT_DIR}/local.params"
173 CONFIG_DIR="local_config_dir"
174 DUMP_CONFIG="no"
175 LOG_LEVEL="info"
176 CONTROLLER_EXT_SSL_PORT=443
177 TESTS_DIR="tests"
178
179 NGINX_INSTALL_SOURCE="install_from_repo"
180
181 CLUSTER=""
182 DOMAIN=""
183
184 # Hostnames/IPs used for single-host deploys
185 IP_INT="127.0.1.1"
186
187 # Initial user setup
188 INITIAL_USER=""
189 INITIAL_USER_EMAIL=""
190 INITIAL_USER_PASSWORD=""
191
192 CONTROLLER_EXT_SSL_PORT=8000
193 KEEP_EXT_SSL_PORT=25101
194 # Both for collections and downloads
195 KEEPWEB_EXT_SSL_PORT=9002
196 WEBSHELL_EXT_SSL_PORT=4202
197 WEBSOCKET_EXT_SSL_PORT=8002
198 WORKBENCH1_EXT_SSL_PORT=443
199 WORKBENCH2_EXT_SSL_PORT=3001
200
201 SSL_MODE="self-signed"
202 USE_LETSENCRYPT_ROUTE53="no"
203 CUSTOM_CERTS_DIR="${SCRIPT_DIR}/local_config_dir/certs"
204
205 ## These are ARVADOS-related parameters
206 # For a stable release, change RELEASE "production" and VERSION to the
207 # package version (including the iteration, e.g. X.Y.Z-1) of the
208 # release.
209 # The "local.params.example.*" files already set "RELEASE=production"
210 # to deploy  production-ready packages
211 RELEASE="development"
212 VERSION="latest"
213
214 # These are arvados-formula-related parameters
215 # An arvados-formula tag. For a stable release, this should be a
216 # branch name (e.g. X.Y-dev) or tag for the release.
217 # ARVADOS_TAG="2.2.0"
218 # BRANCH="main"
219
220 # Other formula versions we depend on
221 POSTGRES_TAG="v0.44.0"
222 NGINX_TAG="v2.8.1"
223 DOCKER_TAG="v2.4.2"
224 LOCALE_TAG="v0.3.4"
225 LETSENCRYPT_TAG="v2.1.0"
226
227 # Salt's dir
228 DUMP_SALT_CONFIG_DIR=""
229 ## states
230 S_DIR="/srv/salt"
231 ## formulas
232 F_DIR="/srv/formulas"
233 ## pillars
234 P_DIR="/srv/pillars"
235 ## tests
236 T_DIR="/tmp/cluster_tests"
237
238 arguments ${@}
239
240 declare -A NODES
241
242 if [ -s ${CONFIG_FILE} ]; then
243   source ${CONFIG_FILE}
244 else
245   echo >&2 "You don't seem to have a config file with initial values."
246   echo >&2 "Please create a '${CONFIG_FILE}' file as described in"
247   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
248   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
249   exit 1
250 fi
251
252 if [ ! -d ${CONFIG_DIR} ]; then
253   echo >&2 "You don't seem to have a config directory with pillars and states."
254   echo >&2 "Please create a '${CONFIG_DIR}' directory (as configured in your '${CONFIG_FILE}'). Please see"
255   echo >&2 "  * https://doc.arvados.org/install/salt-single-host.html#single_host, or"
256   echo >&2 "  * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames"
257   exit 1
258 fi
259
260 if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_DIR} ; then
261   echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified."
262   echo >&2 "Please, fix them and re-run the provision script."
263   exit 1
264 fi
265
266 if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
267   echo >&2 "ERROR: <CLUSTER> must be exactly 5 lowercase alphanumeric characters long"
268   echo >&2 "Fix the cluster name in the 'local.params' file and re-run the provision script"
269   exit 1
270 fi
271
272 # Only used in single_host/single_name deploys
273 if [ ! -z "${HOSTNAME_EXT}" ] ; then
274   # We need to add some extra control vars to manage a single certificate vs. multiple
275   USE_SINGLE_HOSTNAME="yes"
276   # Make sure that the value configured as IP_INT is a real IP on the system.
277   # If we don't error out early here when there is a mismatch, the formula will
278   # fail with hard to interpret nginx errors later on.
279   ip addr list |grep "${IP_INT}/" >/dev/null
280   if [[ $? -ne 0 ]]; then
281     echo "Unable to find the IP_INT address '${IP_INT}' on the system, please correct the value in local.params. Exiting..."
282     exit 1
283   fi
284 else
285   USE_SINGLE_HOSTNAME="no"
286   # We set this variable, anyway, so sed lines do not fail and we don't need to add more
287   # conditionals
288   HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
289 fi
290
291 if [ "${DUMP_CONFIG}" = "yes" ]; then
292   echo "The provision installer will just dump a config under ${DUMP_SALT_CONFIG_DIR} and exit"
293 else
294   # Install a few dependency packages
295   # First, let's figure out the OS we're working on
296   OS_ID=$(grep ^ID= /etc/os-release |cut -f 2 -d=  |cut -f 2 -d \")
297   echo "Detected distro: ${OS_ID}"
298
299   case ${OS_ID} in
300     "centos")
301       echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
302       sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
303       setenforce permissive
304       yum install -y  curl git jq
305       ;;
306     "debian"|"ubuntu")
307       # Wait 2 minutes for any apt locks to clear
308       # This option is supported from apt 1.9.1 and ignored in older apt versions.
309       # Cf. https://blog.sinjakli.co.uk/2021/10/25/waiting-for-apt-locks-without-the-hacky-bash-scripts/
310       DEBIAN_FRONTEND=noninteractive apt -o DPkg::Lock::Timeout=120 update
311       DEBIAN_FRONTEND=noninteractive apt install -y curl git jq
312       ;;
313   esac
314
315   if which salt-call; then
316     echo "Salt already installed"
317   else
318     curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
319     sh /tmp/bootstrap_salt.sh -XdfP -x python3
320     /bin/systemctl stop salt-minion.service
321     /bin/systemctl disable salt-minion.service
322   fi
323
324   # Set salt to masterless mode
325   cat > /etc/salt/minion << EOFSM
326 failhard: "True"
327
328 file_client: local
329 file_roots:
330   base:
331     - ${S_DIR}
332     - ${F_DIR}/*
333
334 pillar_roots:
335   base:
336     - ${P_DIR}
337 EOFSM
338 fi
339
340 mkdir -p ${S_DIR} ${F_DIR} ${P_DIR} ${T_DIR}
341
342 # Get the formula and dependencies
343 cd ${F_DIR} || exit 1
344 echo "Cloning formulas"
345 rm -rf ${F_DIR}/* || exit 1
346 git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
347 ( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
348
349 echo "...locale"
350 git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
351 ( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
352
353 echo "...nginx"
354 git clone --quiet https://github.com/saltstack-formulas/nginx-formula.git ${F_DIR}/nginx
355 ( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
356
357 echo "...postgres"
358 git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
359 ( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
360
361 echo "...letsencrypt"
362 git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
363 ( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
364
365 echo "...arvados"
366 git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
367
368 # If we want to try a specific branch of the formula
369 if [ "x${BRANCH}" != "x" -a $(git rev-parse --abbrev-ref HEAD) != "${BRANCH}" ]; then
370   ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
371 elif [ "x${ARVADOS_TAG}" != "x" -a $(git rev-parse --abbrev-ref HEAD) != "${ARVADOS_TAG}" ]; then
372 ( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
373 fi
374
375 if [ "x${VAGRANT}" = "xyes" ]; then
376   EXTRA_STATES_DIR="/home/vagrant/${CONFIG_DIR}/states"
377   SOURCE_PILLARS_DIR="/home/vagrant/${CONFIG_DIR}/pillars"
378   SOURCE_TESTS_DIR="/home/vagrant/${TESTS_DIR}"
379 else
380   EXTRA_STATES_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/states"
381   SOURCE_PILLARS_DIR="${SCRIPT_DIR}/${CONFIG_DIR}/pillars"
382   SOURCE_TESTS_DIR="${SCRIPT_DIR}/${TESTS_DIR}"
383 fi
384
385 SOURCE_STATES_DIR="${EXTRA_STATES_DIR}"
386
387 echo "Writing pillars and states"
388
389 # Replace variables (cluster,  domain, etc) in the pillars, states and tests
390 # to ease deployment for newcomers
391 if [ ! -d "${SOURCE_PILLARS_DIR}" ]; then
392   echo "${SOURCE_PILLARS_DIR} does not exist or is not a directory. Exiting."
393   exit 1
394 fi
395 for f in $(ls "${SOURCE_PILLARS_DIR}"/*); do
396   sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
397        s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
398        s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
399        s#__CLUSTER__#${CLUSTER}#g;
400        s#__DOMAIN__#${DOMAIN}#g;
401        s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
402        s#__IP_INT__#${IP_INT}#g;
403        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
404        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
405        s#__INITIAL_USER__#${INITIAL_USER}#g;
406        s#__LE_AWS_REGION__#${LE_AWS_REGION}#g;
407        s#__LE_AWS_SECRET_ACCESS_KEY__#${LE_AWS_SECRET_ACCESS_KEY}#g;
408        s#__LE_AWS_ACCESS_KEY_ID__#${LE_AWS_ACCESS_KEY_ID}#g;
409        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
410        s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
411        s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
412        s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g;
413        s#__RELEASE__#${RELEASE}#g;
414        s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g;
415        s#__VERSION__#${VERSION}#g;
416        s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
417        s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
418        s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
419        s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
420        s#__CLUSTER_INT_CIDR__#${CLUSTER_INT_CIDR}#g;
421        s#__CONTROLLER_INT_IP__#${CONTROLLER_INT_IP}#g;
422        s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
423        s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
424        s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
425        s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
426        s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
427        s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
428        s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
429        s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
430        s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
431        s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
432        s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
433   "${f}" > "${P_DIR}"/$(basename "${f}")
434 done
435
436 if [ ! -d "${SOURCE_TESTS_DIR}" ]; then
437   echo "WARNING: The tests directory was not copied to \"${SOURCE_TESTS_DIR}\"."
438   if [ "x${TEST}" = "xyes" ]; then
439     echo "WARNING: Disabling tests for this installation."
440   fi
441   TEST="no"
442 else
443   mkdir -p ${T_DIR}
444   # Replace cluster and domain name in the test files
445   for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
446     FILTERS="s#__CLUSTER__#${CLUSTER}#g;
447          s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
448          s#__DOMAIN__#${DOMAIN}#g;
449          s#__IP_INT__#${IP_INT}#g;
450          s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
451          s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g
452          s#__INITIAL_USER__#${INITIAL_USER}#g;
453          s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
454          s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g"
455     if [ "$USE_SINGLE_HOSTNAME" = "yes" ]; then
456       FILTERS="s#__CLUSTER__.__DOMAIN__#${HOSTNAME_EXT}#g;
457          $FILTERS"
458     fi
459     sed "$FILTERS" \
460       "${f}" > ${T_DIR}/$(basename "${f}")
461   done
462   chmod 755 ${T_DIR}/run-test.sh
463 fi
464
465 # Replace helper state files that differ from the formula's examples
466 if [ -d "${SOURCE_STATES_DIR}" ]; then
467   mkdir -p "${F_DIR}"/extra/extra
468
469   for f in $(ls "${SOURCE_STATES_DIR}"/*); do
470     sed "s#__ANONYMOUS_USER_TOKEN__#${ANONYMOUS_USER_TOKEN}#g;
471          s#__CLUSTER__#${CLUSTER}#g;
472          s#__BLOB_SIGNING_KEY__#${BLOB_SIGNING_KEY}#g;
473          s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
474          s#__DOMAIN__#${DOMAIN}#g;
475          s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
476          s#__IP_INT__#${IP_INT}#g;
477          s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
478          s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
479          s#__INITIAL_USER__#${INITIAL_USER}#g;
480          s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
481          s#__KEEPWEB_EXT_SSL_PORT__#${KEEPWEB_EXT_SSL_PORT}#g;
482          s#__KEEP_EXT_SSL_PORT__#${KEEP_EXT_SSL_PORT}#g;
483          s#__MANAGEMENT_TOKEN__#${MANAGEMENT_TOKEN}#g;
484          s#__RELEASE__#${RELEASE}#g;
485          s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g;
486          s#__VERSION__#${VERSION}#g;
487          s#__CLUSTER_INT_CIDR__#${CLUSTER_INT_CIDR}#g;
488          s#__CONTROLLER_INT_IP__#${CONTROLLER_INT_IP}#g;
489          s#__WEBSOCKET_INT_IP__#${WEBSOCKET_INT_IP}#g;
490          s#__KEEP_INT_IP__#${KEEP_INT_IP}#g;
491          s#__KEEPSTORE0_INT_IP__#${KEEPSTORE0_INT_IP}#g;
492          s#__KEEPSTORE1_INT_IP__#${KEEPSTORE1_INT_IP}#g;
493          s#__KEEPWEB_INT_IP__#${KEEPWEB_INT_IP}#g;
494          s#__WEBSHELL_INT_IP__#${WEBSHELL_INT_IP}#g;
495          s#__WORKBENCH1_INT_IP__#${WORKBENCH1_INT_IP}#g;
496          s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
497          s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
498          s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
499          s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
500          s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
501          s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
502          s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
503          s#__WORKBENCH_SECRET_KEY__#${WORKBENCH_SECRET_KEY}#g" \
504     "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}")
505   done
506 fi
507
508 # Now, we build the SALT states/pillars trees
509 # As we need to separate both states and pillars in case we want specific
510 # roles, we iterate on both at the same time
511
512 # States
513 cat > ${S_DIR}/top.sls << EOFTSLS
514 base:
515   '*':
516     - locale
517 EOFTSLS
518
519 # Pillars
520 cat > ${P_DIR}/top.sls << EOFPSLS
521 base:
522   '*':
523     - locale
524     - arvados
525 EOFPSLS
526
527 # States, extra states
528 if [ -d "${F_DIR}"/extra/extra ]; then
529   SKIP_SNAKE_OIL="snakeoil_certs"
530
531   if [[ "$DEV_MODE" = "yes" || "${SSL_MODE}" == "self-signed" ]] ; then
532     # In dev mode, we create some snake oil certs that we'll
533     # use as CUSTOM_CERTS, so we don't skip the states file.
534     # Same when using self-signed certificates.
535     SKIP_SNAKE_OIL="dont_add_snakeoil_certs"
536   fi
537   for f in $(ls "${F_DIR}"/extra/extra/*.sls | egrep -v "${SKIP_SNAKE_OIL}|shell_"); do
538   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
539   done
540   # Use byo or self-signed certificates
541   if [ "${SSL_MODE}" != "lets-encrypt" ]; then
542     mkdir -p "${F_DIR}"/extra/extra/files
543   fi
544 fi
545
546 # If we want specific roles for a node, just add the desired states
547 # and its dependencies
548 if [ -z "${ROLES}" ]; then
549   # States
550   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
551   if [ "${SSL_MODE}" = "lets-encrypt" ]; then
552     if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
553       grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - extra.aws_credentials" >> ${S_DIR}/top.sls
554     fi
555     grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
556   else
557     mkdir -p /srv/salt/certs
558     if [ "${SSL_MODE}" = "bring-your-own" ]; then
559       # Copy certs to formula extra/files
560       cp -rv ${CUSTOM_CERTS_DIR}/* /srv/salt/certs/
561       # We add the custom_certs state
562       grep -q "custom_certs"    ${S_DIR}/top.sls || echo "    - extra.custom_certs" >> ${S_DIR}/top.sls
563     fi
564     # In self-signed mode, the certificate files will be created and put in the
565     # destination directory by the snakeoil_certs.sls state file
566   fi
567
568   echo "    - postgres" >> ${S_DIR}/top.sls
569   echo "    - docker.software" >> ${S_DIR}/top.sls
570   echo "    - arvados" >> ${S_DIR}/top.sls
571   echo "    - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
572   echo "    - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
573   echo "    - extra.passenger_rvm" >> ${S_DIR}/top.sls
574
575   # Pillars
576   echo "    - docker" >> ${P_DIR}/top.sls
577   echo "    - nginx_api_configuration" >> ${P_DIR}/top.sls
578   echo "    - nginx_controller_configuration" >> ${P_DIR}/top.sls
579   echo "    - nginx_keepproxy_configuration" >> ${P_DIR}/top.sls
580   echo "    - nginx_keepweb_configuration" >> ${P_DIR}/top.sls
581   echo "    - nginx_passenger" >> ${P_DIR}/top.sls
582   echo "    - nginx_websocket_configuration" >> ${P_DIR}/top.sls
583   echo "    - nginx_webshell_configuration" >> ${P_DIR}/top.sls
584   echo "    - nginx_workbench2_configuration" >> ${P_DIR}/top.sls
585   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
586   echo "    - postgresql" >> ${P_DIR}/top.sls
587
588   # We need to tweak the Nginx's pillar depending whether we want plan nginx or nginx+passenger
589   NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
590   sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
591
592   if [ "${SSL_MODE}" = "lets-encrypt" ]; then
593     if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
594       grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
595     fi
596     grep -q "letsencrypt" ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
597
598     hosts=("controller" "websocket" "workbench" "workbench2" "webshell" "keepproxy")
599     if [ ${USE_SINGLE_HOSTNAME} = "no" ]; then
600       hosts+=("download" "collections")
601     else
602       hosts+=("keepweb")
603     fi
604
605     for c in "${hosts[@]}"; do
606       # Are we in a single-host-single-hostname env?
607       if [ "${USE_SINGLE_HOSTNAME}" = "yes" ]; then
608         # Are we in a single-host-single-hostname env?
609         CERT_NAME=${HOSTNAME_EXT}
610       else
611         # We are in a multiple-hostnames env
612         CERT_NAME=${c}.${CLUSTER}.${DOMAIN}
613       fi
614
615       # As the pillar differs whether we use LE or custom certs, we need to do a final edition on them
616       sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${CERT_NAME}*/g;
617               s#__CERT_PEM__#/etc/letsencrypt/live/${CERT_NAME}/fullchain.pem#g;
618               s#__CERT_KEY__#/etc/letsencrypt/live/${CERT_NAME}/privkey.pem#g" \
619       ${P_DIR}/nginx_${c}_configuration.sls
620     done
621   else
622     # Use custom certs (either dev mode or prod)
623     grep -q "extra_custom_certs" ${P_DIR}/top.sls || echo "    - extra_custom_certs" >> ${P_DIR}/top.sls
624     # And add the certs in the custom_certs pillar
625     echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
626     echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
627
628     for c in controller websocket workbench workbench2 webshell keepweb keepproxy; do
629       # Are we in a single-host-single-hostname env?
630       if [ "${USE_SINGLE_HOSTNAME}" = "yes" ]; then
631         # Are we in a single-host-single-hostname env?
632         CERT_NAME=${HOSTNAME_EXT}
633       else
634         # We are in a multiple-hostnames env
635         CERT_NAME=${c}
636       fi
637
638       if [[ "$SSL_MODE" == "bring-your-own" ]]; then
639         copy_custom_cert ${CUSTOM_CERTS_DIR} ${CERT_NAME}
640       fi
641
642       grep -q ${CERT_NAME} ${P_DIR}/extra_custom_certs.sls || echo "  - ${CERT_NAME}" >> ${P_DIR}/extra_custom_certs.sls
643
644       # As the pillar differs whether we use LE or custom certs, we need to do a final edition on them
645       sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${CERT_NAME}.pem/g;
646               s#__CERT_PEM__#/etc/nginx/ssl/arvados-${CERT_NAME}.pem#g;
647               s#__CERT_KEY__#/etc/nginx/ssl/arvados-${CERT_NAME}.key#g" \
648       ${P_DIR}/nginx_${c}_configuration.sls
649     done
650   fi
651 else
652   # If we add individual roles, make sure we add the repo first
653   echo "    - arvados.repo" >> ${S_DIR}/top.sls
654   # We add the extra_custom_certs state
655   grep -q "extra.custom_certs"    ${S_DIR}/top.sls || echo "    - extra.custom_certs" >> ${S_DIR}/top.sls
656
657   # And we add the basic part for the certs pillar
658   if [ "${SSL_MODE}" != "lets-encrypt" ]; then
659     # And add the certs in the custom_certs pillar
660     echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
661     echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
662     grep -q "extra_custom_certs" ${P_DIR}/top.sls || echo "    - extra_custom_certs" >> ${P_DIR}/top.sls
663   fi
664
665   for R in ${ROLES}; do
666     case "${R}" in
667       "database")
668         # States
669         echo "    - postgres" >> ${S_DIR}/top.sls
670         # Pillars
671         echo '    - postgresql' >> ${P_DIR}/top.sls
672       ;;
673       "api")
674         # States
675         # FIXME: https://dev.arvados.org/issues/17352
676         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
677         if grep -q "    - nginx.*$" ${S_DIR}/top.sls; then
678           sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${S_DIR}/top.sls
679         else
680           echo "    - nginx.passenger" >> ${S_DIR}/top.sls
681         fi
682         echo "    - extra.passenger_rvm" >> ${S_DIR}/top.sls
683         ### If we don't install and run LE before arvados-api-server, it fails and breaks everything
684         ### after it. So we add this here as we are, after all, sharing the host for api and controller
685         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
686           if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
687             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
688           fi
689           grep -q "letsencrypt" ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
690         else
691           # Use custom certs
692           if [ "${SSL_MODE}" = "bring-your-own" ]; then
693             copy_custom_cert ${CUSTOM_CERTS_DIR} controller
694           fi
695           grep -q controller ${P_DIR}/extra_custom_certs.sls || echo "  - controller" >> ${P_DIR}/extra_custom_certs.sls
696         fi
697         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
698         # Pillars
699         grep -q "aws_credentials" ${P_DIR}/top.sls          || echo "    - aws_credentials" >> ${P_DIR}/top.sls
700         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
701         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
702         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
703
704         # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
705         NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
706         sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
707       ;;
708       "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
709         NGINX_INSTALL_SOURCE="install_from_repo"
710         # States
711         if [ "${R}" = "workbench" ]; then
712           NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
713           if grep -q "    - nginx$" ${S_DIR}/top.sls; then
714             sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${S_DIR}/top.sls
715           else
716             echo "    - nginx.passenger" >> ${S_DIR}/top.sls
717           fi
718         else
719           grep -q "nginx" ${S_DIR}/top.sls || echo "    - nginx" >> ${S_DIR}/top.sls
720         fi
721         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
722           if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
723             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
724           fi
725           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
726         else
727           # Use custom certs, special case for keepweb
728           if [ ${R} = "keepweb" ]; then
729             if [ "${SSL_MODE}" = "bring-your-own" ]; then
730               copy_custom_cert ${CUSTOM_CERTS_DIR} download
731               copy_custom_cert ${CUSTOM_CERTS_DIR} collections
732             fi
733           else
734             if [ "${SSL_MODE}" = "bring-your-own" ]; then
735               copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
736             fi
737           fi
738         fi
739         # webshell role is just a nginx vhost, so it has no state
740         if [ "${R}" != "webshell" ]; then
741           grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
742         fi
743         # Pillars
744         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
745         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
746         # Special case for keepweb
747         if [ ${R} = "keepweb" ]; then
748           grep -q "nginx_download_configuration" ${P_DIR}/top.sls || echo "    - nginx_download_configuration" >> ${P_DIR}/top.sls
749           grep -q "nginx_collections_configuration" ${P_DIR}/top.sls || echo "    - nginx_collections_configuration" >> ${P_DIR}/top.sls
750         fi
751
752         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
753           if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
754             grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
755           fi
756           grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
757           grep -q "letsencrypt_${R}_configuration" ${P_DIR}/top.sls || echo "    - letsencrypt_${R}_configuration" >> ${P_DIR}/top.sls
758
759           # As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
760           # Special case for keepweb
761           if [ ${R} = "keepweb" ]; then
762             for kwsub in download collections; do
763               sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${kwsub}.${CLUSTER}.${DOMAIN}*/g;
764                       s#__CERT_PEM__#/etc/letsencrypt/live/${kwsub}.${CLUSTER}.${DOMAIN}/fullchain.pem#g;
765                       s#__CERT_KEY__#/etc/letsencrypt/live/${kwsub}.${CLUSTER}.${DOMAIN}/privkey.pem#g" \
766               ${P_DIR}/nginx_${kwsub}_configuration.sls
767             done
768           else
769             sed -i "s/__CERT_REQUIRES__/cmd: create-initial-cert-${R}.${CLUSTER}.${DOMAIN}*/g;
770                     s#__CERT_PEM__#/etc/letsencrypt/live/${R}.${CLUSTER}.${DOMAIN}/fullchain.pem#g;
771                     s#__CERT_KEY__#/etc/letsencrypt/live/${R}.${CLUSTER}.${DOMAIN}/privkey.pem#g" \
772             ${P_DIR}/nginx_${R}_configuration.sls
773           fi
774         else
775           # As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
776           # Special case for keepweb
777           if [ ${R} = "keepweb" ]; then
778             for kwsub in download collections; do
779               sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${kwsub}.pem/g;
780                       s#__CERT_PEM__#/etc/nginx/ssl/arvados-${kwsub}.pem#g;
781                       s#__CERT_KEY__#/etc/nginx/ssl/arvados-${kwsub}.key#g" \
782               ${P_DIR}/nginx_${kwsub}_configuration.sls
783               grep -q ${kwsub} ${P_DIR}/extra_custom_certs.sls || echo "  - ${kwsub}" >> ${P_DIR}/extra_custom_certs.sls
784             done
785           else
786             sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${R}.pem/g;
787                     s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g;
788                     s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \
789             ${P_DIR}/nginx_${R}_configuration.sls
790             grep -q ${R}$ ${P_DIR}/extra_custom_certs.sls || echo "  - ${R}" >> ${P_DIR}/extra_custom_certs.sls
791           fi
792         fi
793         # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
794         sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
795       ;;
796       "shell")
797         # States
798         echo "    - extra.shell_sudo_passwordless" >> ${S_DIR}/top.sls
799         echo "    - extra.shell_cron_add_login_sync" >> ${S_DIR}/top.sls
800         grep -q "docker" ${S_DIR}/top.sls       || echo "    - docker.software" >> ${S_DIR}/top.sls
801         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
802         # Pillars
803         grep -q "docker" ${P_DIR}/top.sls       || echo "    - docker" >> ${P_DIR}/top.sls
804       ;;
805       "dispatcher" | "keepbalance" | "keepstore")
806         # States
807         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
808         # Pillars
809         # ATM, no specific pillar needed
810       ;;
811       *)
812         echo "Unknown role ${R}"
813         exit 1
814       ;;
815     esac
816   done
817 fi
818
819 if [ "${DUMP_CONFIG}" = "yes" ]; then
820   # We won't run the rest of the script because we're just dumping the config
821   exit 0
822 fi
823
824 # Now run the install
825 salt-call --local state.apply -l ${LOG_LEVEL}
826
827 # Finally, make sure that /etc/hosts is not overwritten on reboot
828 if [ -d /etc/cloud/cloud.cfg.d ]; then
829   # TODO: will this work on CentOS?
830   sed -i 's/^manage_etc_hosts: true/#manage_etc_hosts: true/g' /etc/cloud/cloud.cfg.d/*
831 fi
832
833 # Leave a copy of the Arvados CA so the user can copy it where it's required
834 if [ "$DEV_MODE" = "yes" ]; then
835   ARVADOS_SNAKEOIL_CA_DEST_FILE="${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem"
836
837   # If running in a vagrant VM, also add default user to docker group
838   if [ "x${VAGRANT}" = "xyes" ]; then
839     echo "Adding the vagrant user to the docker group"
840     usermod -a -G docker vagrant
841     ARVADOS_SNAKEOIL_CA_DEST_FILE="/vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem"
842   fi
843   if [ -f /etc/ssl/certs/arvados-snakeoil-ca.pem ]; then
844     echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
845     cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${ARVADOS_SNAKEOIL_CA_DEST_FILE}
846   fi
847 fi
848
849 # Test that the installation finished correctly
850 if [ "x${TEST}" = "xyes" ]; then
851   cd ${T_DIR}
852   # If we use RVM, we need to run this with it, or most ruby commands will fail
853   RVM_EXEC=""
854   if [ -x /usr/local/rvm/bin/rvm-exec ]; then
855     RVM_EXEC="/usr/local/rvm/bin/rvm-exec"
856   fi
857   ${RVM_EXEC} ./run-test.sh
858 fi