3 # Copyright (C) The Arvados Authors. All rights reserved.
5 # SPDX-License-Identifier: CC-BY-SA-3.0
10 # Helps manage the configuration in a git repository, and then deploy
11 # nodes by pushing a copy of the git repository to each node and
12 # running the provision script to do the actual installation and
20 declare CONFIG_FILE=local.params
22 # The salt template directory
23 declare CONFIG_DIR=local_config_dir
25 # The 5-character Arvados cluster id
26 # This will be populated by loadconfig()
29 # The parent domain (not including the cluster id)
30 # This will be populated by loadconfig()
33 # A bash associative array listing each node and mapping to the roles
34 # that should be provisioned on those nodes.
35 # This will be populated by loadconfig()
38 # A bash associative array listing each role and mapping to the nodes
39 # that should be provisioned with this role.
40 # This will be populated by loadconfig()
43 # The ssh user we'll use
44 # This will be populated by loadconfig()
47 # The git repository that we'll push to on all the nodes
48 # This will be populated by loadconfig()
51 # The public host used as an SSH jump host
52 # This will be populated by loadconfig()
53 declare USE_SSH_JUMPHOST
55 # The temp file that will get used to disable envvar forwarding to avoid locale
56 # issues in Debian distros.
57 # This will be populated by loadconfig()
67 if [[ -n "$MISSING" ]]; then
68 echo "Some tools are missing, please make sure you have the 'git' and 'iproute2' packages installed"
75 local SSH=$(ssh_cmd "$NODE")
76 # Delete the old repository
77 $SSH $DEPLOY_USER@$NODE rm -rf ${GITTARGET}.git ${GITTARGET}
84 # Synchronizes the configuration by creating a git repository on
85 # each node, pushing our branch, and updating the checkout.
87 if [[ "$NODE" != localhost ]]; then
88 SSH=$(ssh_cmd "$NODE")
89 GIT="eval $(git_cmd $NODE)"
93 # Update the git remote for the remote repository.
94 if ! $GIT remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git; then
95 $GIT remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git
98 # Initialize the git repository. We're
99 # actually going to make two repositories here because git
100 # will complain if you try to push to a repository with a
101 # checkout. So we're going to create a "bare" repository
102 # and then clone a regular repository (with a checkout)
105 $SSH $DEPLOY_USER@$NODE git init --bare --shared=0600 ${GITTARGET}.git
106 $GIT push $NODE $BRANCH
107 $SSH $DEPLOY_USER@$NODE "umask 0077 && git clone -s ${GITTARGET}.git ${GITTARGET} && git -C ${GITTARGET} checkout ${BRANCH}"
116 # Deploy a node. This runs the provision script on the node, with
117 # the appropriate roles.
121 if [[ -z "$ROLES" ]]; then
122 echo "No roles specified for $NODE, will deploy all roles"
124 ROLES="--roles ${ROLES}"
127 logfile=deploy-${NODE}-$(date -Iseconds).log
128 SSH=$(ssh_cmd "$NODE")
130 if [[ "$NODE" = localhost ]]; then
132 if [[ $(whoami) != 'root' ]]; then
135 $SUDO ./provision.sh --config ${CONFIG_FILE} ${ROLES} 2>&1 | tee $logfile
137 $SSH $DEPLOY_USER@$NODE "cd ${GITTARGET} && git log -n1 HEAD && DISABLED_CONTROLLER=\"$DISABLED_CONTROLLER\" sudo --preserve-env=DISABLED_CONTROLLER ./provision.sh --config ${CONFIG_FILE} ${ROLES}" 2>&1 | tee $logfile
144 local CERTPATH="${CONFIG_DIR}/certs/${CERTNAME}"
145 if [[ ! -f "${CERTPATH}.crt" || ! -e "${CERTPATH}.key" ]]; then
146 echo "Missing ${CERTPATH}.crt or ${CERTPATH}.key files"
152 if ! [[ -s ${CONFIG_FILE} && -s ${CONFIG_FILE}.secrets ]]; then
153 echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?"
156 GITTARGET=arvados-deploy-config-${CLUSTER}
158 # Set up SSH so that it doesn't forward any environment variable. This is to avoid
159 # getting "setlocale" errors on the first run, depending on the distro being used
160 # to run the installer (like Debian).
161 SSH_CONFFILE=$(mktemp)
162 echo "Include config SendEnv -*" >${SSH_CONFFILE}
167 if [ -z "${USE_SSH_JUMPHOST}" -o "${NODE}" == "${USE_SSH_JUMPHOST}" -o "${NODE}" == "localhost" ]; then
168 echo "ssh -F ${SSH_CONFFILE}"
170 echo "ssh -F ${SSH_CONFFILE} -J ${DEPLOY_USER}@${USE_SSH_JUMPHOST}"
176 echo "GIT_SSH_COMMAND=\"$(ssh_cmd ${NODE})\" git"
183 if [[ -n "$subcmd" ]]; then
188 if [[ ! -f provision.sh ]]; then
189 echo "Must be run from arvados/tools/salt-install"
203 if [[ -z "$PARAMS" || ! -f local.params.example.$PARAMS ]]; then
204 echo "Not found: local.params.example.$PARAMS"
205 echo "Expected one of multiple_hosts, single_host_multiple_hostnames, single_host_single_hostname"
209 if [[ -z "$SLS" || ! -d config_examples/$SLS ]]; then
210 echo "Not found: config_examples/$SLS"
211 echo "Expected one of multi_host/aws, single_host/multiple_hostnames, single_host/single_hostname"
215 if [[ -z "$SETUPDIR" || -z "$PARAMS" || -z "$SLS" ]]; then
216 echo "installer.sh <setup dir to initialize> <params template> <config template>"
220 if [[ -n "$err" ]]; then
224 echo "Initializing $SETUPDIR"
225 git init --shared=0600 $SETUPDIR
226 cp -r *.sh tests $SETUPDIR
228 cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE}
229 cp local.params.secrets.example $SETUPDIR/${CONFIG_FILE}.secrets
230 cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR}
232 if [[ -n "$TERRAFORM" ]]; then
233 mkdir $SETUPDIR/terraform
234 cp -r $TERRAFORM/* $SETUPDIR/terraform/
238 echo '*.log' >.gitignore
239 echo '**/.terraform' >>.gitignore
240 echo '**/.infracost' >>.gitignore
242 if [[ -n "$TERRAFORM" ]]; then
246 git add *.sh ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} tests .gitignore
247 git commit -m"initial commit"
250 echo "Setup directory $SETUPDIR initialized."
251 if [[ -n "$TERRAFORM" ]]; then
252 (cd $SETUPDIR/terraform/vpc && terraform init)
253 (cd $SETUPDIR/terraform/data-storage && terraform init)
254 (cd $SETUPDIR/terraform/services && terraform init)
255 echo "Now go to $SETUPDIR, customize 'terraform/vpc/terraform.tfvars' as needed, then run 'installer.sh terraform'"
257 echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}', '${CONFIG_FILE}.secrets' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'"
262 logfile=terraform-$(date -Iseconds).log
263 (cd terraform/vpc && terraform apply -auto-approve) 2>&1 | tee -a $logfile
264 (cd terraform/data-storage && terraform apply -auto-approve) 2>&1 | tee -a $logfile
265 (cd terraform/services && terraform apply -auto-approve) 2>&1 | grep -v letsencrypt_iam_secret_access_key | tee -a $logfile
266 (cd terraform/services && echo -n 'letsencrypt_iam_secret_access_key = ' && terraform output letsencrypt_iam_secret_access_key) 2>&1 | tee -a $logfile
270 logfile=terraform-$(date -Iseconds).log
271 (cd terraform/services && terraform destroy) 2>&1 | tee -a $logfile
272 (cd terraform/data-storage && terraform destroy) 2>&1 | tee -a $logfile
273 (cd terraform/vpc && terraform destroy) 2>&1 | tee -a $logfile
277 for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do
279 tr -dc A-Za-z0-9 </dev/urandom | head -c 32
294 if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR}; then
296 echo "Some parameters still need to be updated. Please fix them and then re-run deploy."
300 if [[ -z "${DATABASE_POSTGRESQL_VERSION:-}" ]]; then
302 echo "Please configure DATABASE_POSTGRESQL_VERSION in local.params: It should match the version of the PostgreSQL service you're going to use."
306 if [[ ${SSL_MODE} == "bring-your-own" ]]; then
307 if [[ ! -z "${ROLE2NODES['balancer']:-}" ]]; then
310 if [[ ! -z "${ROLE2NODES['controller']:-}" ]]; then
313 if [[ ! -z "${ROLE2NODES['keepproxy']:-}" ]]; then
316 if [[ ! -z "${ROLE2NODES['keepweb']:-}" ]]; then
317 checkcert collections
320 if [[ ! -z "${ROLE2NODES['monitoring']:-}" ]]; then
324 if [[ ! -z "${ROLE2NODES['webshell']:-}" ]]; then
327 if [[ ! -z "${ROLE2NODES['websocket']:-}" ]]; then
330 if [[ ! -z "${ROLE2NODES['workbench']:-}" ]]; then
333 if [[ ! -z "${ROLE2NODES['workbench2']:-}" ]]; then
338 BRANCH=$(git rev-parse --abbrev-ref HEAD)
343 if ! git diff --cached --exit-code --quiet; then
344 git commit -m"prepare for deploy"
347 # Used for rolling updates to disable individual nodes at the
349 export DISABLED_CONTROLLER=""
350 if [[ -z "$NODE" ]]; then
351 for NODE in "${!NODES[@]}"; do
352 # First, just confirm we can ssh to each node.
353 $(ssh_cmd "$NODE") $DEPLOY_USER@$NODE true
356 for NODE in "${!NODES[@]}"; do
357 # Do 'database' role first,
358 if [[ "${NODES[$NODE]}" =~ database ]]; then
359 deploynode $NODE "${NODES[$NODE]}" $BRANCH
364 BALANCER=${ROLE2NODES['balancer']:-}
366 # Check if there are multiple controllers, they'll be comma-separated
368 if [[ ${ROLE2NODES['controller']} =~ , ]]; then
369 # If we have multiple controllers then there must be
370 # load balancer. We want to do a rolling update, take
371 # down each node at the load balancer before updating
374 for NODE in "${!NODES[@]}"; do
375 if [[ "${NODES[$NODE]}" =~ controller ]]; then
376 export DISABLED_CONTROLLER=$NODE
378 # Update balancer that the node is disabled
379 deploynode $BALANCER "${NODES[$BALANCER]}" $BRANCH
381 # Now update the node itself
382 deploynode $NODE "${NODES[$NODE]}" $BRANCH
387 # Only one controller, check if it wasn't already taken care of.
388 NODE=${ROLE2NODES['controller']}
389 if [[ ! -z "${NODES[$NODE]:-}" ]]; then
390 deploynode $NODE "${NODES[$NODE]}" $BRANCH
395 if [[ -n "$BALANCER" ]]; then
396 # Deploy balancer. In the rolling update case, this
397 # will re-enable all the controllers at the balancer.
398 export DISABLED_CONTROLLER=""
399 deploynode $BALANCER "${NODES[$BALANCER]}" $BRANCH
400 unset NODES[$BALANCER]
403 for NODE in "${!NODES[@]}"; do
404 # Everything else (we removed the nodes that we
405 # already deployed from the list)
406 deploynode $NODE "${NODES[$NODE]}" $BRANCH
409 # Just deploy the node that was supplied on the command line.
410 deploynode $NODE "${NODES[$NODE]}" $BRANCH
415 echo "Completed deploy, run 'installer.sh diagnostics' to verify the install"
426 if ! which arvados-client; then
427 echo "arvados-client not found, install 'arvados-client' package with 'apt-get' or 'yum'"
431 if [[ -z "$LOCATION" ]]; then
432 echo "Need to provide '-internal-client' or '-external-client'"
434 echo "-internal-client You are running this on the same private network as the Arvados cluster (e.g. on one of the Arvados nodes)"
435 echo "-external-client You are running this outside the private network of the Arvados cluster (e.g. your workstation)"
439 export ARVADOS_API_HOST="${DOMAIN}:${CONTROLLER_EXT_SSL_PORT}"
440 export ARVADOS_API_TOKEN="$SYSTEM_ROOT_TOKEN"
442 arvados-client diagnostics $LOCATION
446 echo "Arvados installer"
448 echo "initialize initialize the setup directory for configuration"
449 echo "terraform create cloud resources using terraform"
450 echo "terraform-destroy destroy cloud resources created by terraform"
451 echo "generate-tokens generate random values for tokens"
452 echo "deploy deploy the configuration from the setup directory"
453 echo "diagnostics check your install using diagnostics"