X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/093ec98e4a065acfc537ea22c08c337c115fe273..36f5a68fd76e60f2cfdd0cf584d1d2de005e19ec:/tools/salt-install/installer.sh

diff --git a/tools/salt-install/installer.sh b/tools/salt-install/installer.sh
index 534ee629c7..b042d1a835 100755
--- a/tools/salt-install/installer.sh
+++ b/tools/salt-install/installer.sh
@@ -4,141 +4,357 @@
 #
 # SPDX-License-Identifier: CC-BY-SA-3.0
 
-set -e
+#
+# installer.sh
+#
+# Helps manage the configuration in a git repository, and then deploy
+# nodes by pushing a copy of the git repository to each node and
+# running the provision script to do the actual installation and
+# configuration.
+#
+
+set -eu
+set -o pipefail
+
+# The parameter file
+declare CONFIG_FILE=local.params
 
+# The salt template directory
+declare CONFIG_DIR=local_config_dir
+
+# The 5-character Arvados cluster id
+# This will be populated by loadconfig()
+declare CLUSTER
+
+# The parent domain (not including the cluster id)
+# This will be populated by loadconfig()
+declare DOMAIN
+
+# A bash associative array listing each node and mapping to the roles
+# that should be provisioned on those nodes.
+# This will be populated by loadconfig()
 declare -A NODES
 
+# The ssh user we'll use
+# This will be populated by loadconfig()
+declare DEPLOY_USER
+
+# The git repository that we'll push to on all the nodes
+# This will be populated by loadconfig()
+declare GITTARGET
+
+# The public host used as an SSH jump host
+# This will be populated by loadconfig()
+declare USE_SSH_JUMPHOST
+
+checktools() {
+    local MISSING=''
+    for a in git ip ; do
+	if ! which $a ; then
+	    MISSING="$MISSING $a"
+	fi
+    done
+    if [[ -n "$MISSING" ]] ; then
+	echo "Some tools are missing, please make sure you have the 'git' and 'iproute2' packages installed"
+	exit 1
+    fi
+}
+
+cleanup() {
+    local NODE=$1
+    local SSH=`ssh_cmd "$NODE"`
+    # Delete the old repository
+    $SSH $DEPLOY_USER@$NODE rm -rf ${GITTARGET}.git ${GITTARGET}
+}
+
 sync() {
-    if test "$NODE" != localhost ; then
-	if ! ssh $NODE test -d arvados-setup ; then
-	    ssh $NODE git init --bare arvados-setup.git
-	    if ! git remote add $NODE $DEPLOY_USER@$NODE:arvados-setup.git ; then
-		git remote set-url $NODE $DEPLOY_USER@$NODE:arvados-setup.git
-	    fi
-	    git push $NODE $BRANCH
-	    ssh $NODE git clone arvados-setup.git arvados-setup
-	fi
-
-	git push $NODE $BRANCH
-	ssh $NODE git -C arvados-setup checkout $BRANCH
-	ssh $NODE git -C arvados-setup pull
+    local NODE=$1
+    local BRANCH=$2
+
+    # Synchronizes the configuration by creating a git repository on
+    # each node, pushing our branch, and updating the checkout.
+
+    if [[ "$NODE" != localhost ]] ; then
+	SSH=`ssh_cmd "$NODE"`
+	GIT="eval `git_cmd $NODE`"
+
+	cleanup $NODE
+
+	# Update the git remote for the remote repository.
+	if ! $GIT remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git ; then
+	    $GIT remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git
+	fi
+
+	# Initialize the git repository.  We're
+	# actually going to make two repositories here because git
+	# will complain if you try to push to a repository with a
+	# checkout. So we're going to create a "bare" repository
+	# and then clone a regular repository (with a checkout)
+	# from that.
+
+	$SSH $DEPLOY_USER@$NODE git init --bare --shared=0600 ${GITTARGET}.git
+	$GIT push $NODE $BRANCH
+	$SSH $DEPLOY_USER@$NODE "umask 0077 && git clone -s ${GITTARGET}.git ${GITTARGET} && git -C ${GITTARGET} checkout ${BRANCH}"
     fi
 }
 
 deploynode() {
-    if test -z "${NODES[$NODE]}" ; then
-	echo "No roles declared for '$NODE' in local.params"
-	exit 1
+    local NODE=$1
+    local ROLES=$2
+
+    # Deploy a node.  This runs the provision script on the node, with
+    # the appropriate roles.
+
+    if [[ -z "$ROLES" ]] ; then
+		echo "No roles specified for $NODE, will deploy all roles"
+    else
+		ROLES="--roles ${ROLES}"
     fi
 
-    if test $NODE = localhost ; then
-	sudo ./provision.sh --config local.params --roles ${NODES[$NODE]}
+    logfile=deploy-${NODE}-$(date -Iseconds).log
+    SSH=`ssh_cmd "$NODE"`
+
+    if [[ "$NODE" = localhost ]] ; then
+	    SUDO=''
+		if [[ $(whoami) != 'root' ]] ; then
+			SUDO=sudo
+		fi
+		$SUDO ./provision.sh --config ${CONFIG_FILE} ${ROLES} 2>&1 | tee $logfile
     else
-	ssh $DEPLOY_USER@$NODE "cd arvados-setup && sudo ./provision.sh --config local.params --roles ${NODES[$NODE]}"
+	    $SSH $DEPLOY_USER@$NODE "cd ${GITTARGET} && git log -n1 HEAD && sudo ./provision.sh --config ${CONFIG_FILE} ${ROLES}" 2>&1 | tee $logfile
+	    cleanup $NODE
     fi
 }
 
 loadconfig() {
-    CONFIG_FILE=local.params
-    if ! test -s $CONFIG_FILE ; then
-	echo "Must be run from arvados-setup, maybe you need to 'initialize' first?"
+    if [[ -s ${CONFIG_FILE} && -s ${CONFIG_FILE}.secrets ]]; then
+		echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?"
     fi
+    source ${CONFIG_FILE}.secrets
     source ${CONFIG_FILE}
+    GITTARGET=arvados-deploy-config-${CLUSTER}
 }
 
+ssh_cmd() {
+	local NODE=$1
+	if [ -z "${USE_SSH_JUMPHOST}" -o "${NODE}" == "${USE_SSH_JUMPHOST}" -o "${NODE}" == "localhost" ]; then
+		echo "ssh"
+	else
+		echo "ssh -J ${DEPLOY_USER}@${USE_SSH_JUMPHOST}"
+	fi
+}
+
+git_cmd() {
+	local NODE=$1
+	echo "GIT_SSH_COMMAND=\"`ssh_cmd ${NODE}`\" git"
+}
+
+set +u
 subcmd="$1"
-if test -n "$subcmd" ; then
+set -u
+
+if [[ -n "$subcmd" ]] ; then
     shift
 fi
 case "$subcmd" in
     initialize)
-	if ! test -f provision.sh ; then
+	if [[ ! -f provision.sh ]] ; then
 	    echo "Must be run from arvados/tools/salt-install"
 	    exit
 	fi
 
+	checktools
+
+	set +u
 	SETUPDIR=$1
 	PARAMS=$2
 	SLS=$3
+	TERRAFORM=$4
+	set -u
 
 	err=
-	if test -z "$PARAMS" -o ! -f local.params.example.$PARAMS ; then
+	if [[ -z "$PARAMS" || ! -f local.params.example.$PARAMS ]] ; then
 	    echo "Not found: local.params.example.$PARAMS"
 	    echo "Expected one of multiple_hosts, single_host_multiple_hostnames, single_host_single_hostname"
 	    err=1
 	fi
 
-	if test -z "$SLS" -o ! -d config_examples/$SLS ; then
+	if [[ -z "$SLS" || ! -d config_examples/$SLS ]] ; then
 	    echo "Not found: config_examples/$SLS"
 	    echo "Expected one of multi_host/aws, single_host/multiple_hostnames, single_host/single_hostname"
 	    err=1
 	fi
 
-	if test -z "$SETUPDIR" -o -z "$PARAMS" -o -z "$SLS" ; then
+	if [[ -z "$SETUPDIR" || -z "$PARAMS" || -z "$SLS" ]]; then
 	    echo "installer.sh <setup dir to initialize> <params template> <config template>"
 	    err=1
 	fi
 
-	if test -n "$err" ; then
+	if [[ -n "$err" ]] ; then
 	    exit 1
 	fi
 
 	echo "Initializing $SETUPDIR"
-	git init $SETUPDIR
+	git init --shared=0600 $SETUPDIR
 	cp -r *.sh tests $SETUPDIR
 
-	cp local.params.example.$PARAMS $SETUPDIR/local.params
-	cp -r config_examples/$SLS $SETUPDIR/local_config_dir
+	cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE}
+	cp local.params.secrets.example $SETUPDIR/${CONFIG_FILE}.secrets
+	cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR}
+
+	if [[ -n "$TERRAFORM" ]] ; then
+	    mkdir $SETUPDIR/terraform
+	    cp -r $TERRAFORM/* $SETUPDIR/terraform/
+	fi
 
 	cd $SETUPDIR
-	git add *.sh local.params local_config_dir tests
+	echo '*.log' > .gitignore
+	echo '**/.terraform' >> .gitignore
+	echo '**/.infracost' >> .gitignore
+
+	if [[ -n "$TERRAFORM" ]] ; then
+		git add terraform
+	fi
+
+	git add *.sh ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} tests .gitignore
 	git commit -m"initial commit"
 
-	echo "setup directory initialized, now go to $SETUPDIR, edit 'local.params' and 'local_config_dir' as needed, then run 'installer.sh deploy'"
+	echo
+	echo "Setup directory $SETUPDIR initialized."
+	if [[ -n "$TERRAFORM" ]] ; then
+	    (cd $SETUPDIR/terraform/vpc && terraform init)
+	    (cd $SETUPDIR/terraform/data-storage && terraform init)
+	    (cd $SETUPDIR/terraform/services && terraform init)
+	    echo "Now go to $SETUPDIR, customize 'terraform/vpc/terraform.tfvars' as needed, then run 'installer.sh terraform'"
+	else
+		echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}', '${CONFIG_FILE}.secrets' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'"
+	fi
+	;;
+
+    terraform)
+	logfile=terraform-$(date -Iseconds).log
+	(cd terraform/vpc && terraform apply -auto-approve) 2>&1 | tee -a $logfile
+	(cd terraform/data-storage && terraform apply -auto-approve) 2>&1 | tee -a $logfile
+	(cd terraform/services && terraform apply -auto-approve) 2>&1 | grep -v letsencrypt_iam_secret_access_key | tee -a $logfile
+	(cd terraform/services && echo -n 'letsencrypt_iam_secret_access_key = ' && terraform output letsencrypt_iam_secret_access_key) 2>&1 | tee -a $logfile
+	;;
+
+    terraform-destroy)
+	logfile=terraform-$(date -Iseconds).log
+	(cd terraform/services && terraform destroy) 2>&1 | tee -a $logfile
+	(cd terraform/data-storage && terraform destroy) 2>&1 | tee -a $logfile
+	(cd terraform/vpc && terraform destroy) 2>&1 | tee -a $logfile
 	;;
+
+    generate-tokens)
+	for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do
+	    echo ${i}=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 32 ; echo '')
+	done
+	;;
+
     deploy)
+	set +u
 	NODE=$1
+	set -u
+
+	checktools
 
 	loadconfig
 
-	set -x
+	if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} ; then
+	    echo
+	    echo "Some parameters still need to be updated.  Please fix them and then re-run deploy."
+	    exit 1
+	fi
 
-	BRANCH=$(git branch --show-current)
+	BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+	set -x
 
 	git add -A
-	if ! git diff --cached --exit-code ; then
+	if ! git diff --cached --exit-code --quiet ; then
 	    git commit -m"prepare for deploy"
 	fi
 
-	if test -z "$NODE"; then
+	if [[ -z "$NODE" ]]; then
 	    for NODE in "${!NODES[@]}"
 	    do
-		sync
-		deploynode
+		# First, push the git repo to each node.  This also
+		# confirms that we have git and can log into each
+		# node.
+		sync $NODE $BRANCH
+	    done
+
+	    for NODE in "${!NODES[@]}"
+	    do
+		# Do 'database' role first,
+		if [[ "${NODES[$NODE]}" =~ database ]] ; then
+		    deploynode $NODE "${NODES[$NODE]}"
+		    unset NODES[$NODE]
+		fi
+	    done
+
+	    for NODE in "${!NODES[@]}"
+	    do
+		# then  'api' or 'controller' roles
+		if [[ "${NODES[$NODE]}" =~ (api|controller) ]] ; then
+		    deploynode $NODE "${NODES[$NODE]}"
+		    unset NODES[$NODE]
+		fi
+	    done
+
+	    for NODE in "${!NODES[@]}"
+	    do
+		# Everything else (we removed the nodes that we
+		# already deployed from the list)
+		deploynode $NODE "${NODES[$NODE]}"
 	    done
 	else
-	    sync
-	    deploynode
+	    # Just deploy the node that was supplied on the command line.
+	    sync $NODE $BRANCH
+	    deploynode $NODE "${NODES[$NODE]}"
 	fi
 
+	set +x
+	echo
+	echo "Completed deploy, run 'installer.sh diagnostics' to verify the install"
+
 	;;
+
     diagnostics)
 	loadconfig
 
+	set +u
+	declare LOCATION=$1
+	set -u
+
 	if ! which arvados-client ; then
-	    apt-get install arvados-client
+	    echo "arvados-client not found, install 'arvados-client' package with 'apt-get' or 'yum'"
+	    exit 1
 	fi
 
-	export ARVADOS_API_HOST="${CLUSTER}.${DOMAIN}"
+	if [[ -z "$LOCATION" ]] ; then
+	    echo "Need to provide '-internal-client' or '-external-client'"
+	    echo
+	    echo "-internal-client    You are running this on the same private network as the Arvados cluster (e.g. on one of the Arvados nodes)"
+	    echo "-external-client    You are running this outside the private network of the Arvados cluster (e.g. your workstation)"
+	    exit 1
+	fi
+
+	export ARVADOS_API_HOST="${DOMAIN}:${CONTROLLER_EXT_SSL_PORT}"
 	export ARVADOS_API_TOKEN="$SYSTEM_ROOT_TOKEN"
 
-	arvados-client diagnostics -internal-client
+	arvados-client diagnostics $LOCATION
 	;;
+
     *)
 	echo "Arvados installer"
 	echo ""
-	echo "initialize   initialize the setup directory for configuration"
-	echo "deploy       deploy the configuration from the setup directory"
-	echo "diagnostics  check your install using diagnostics"
+	echo "initialize        initialize the setup directory for configuration"
+	echo "terraform         create cloud resources using terraform"
+	echo "terraform-destroy destroy cloud resources created by terraform"
+	echo "generate-tokens   generate random values for tokens"
+	echo "deploy            deploy the configuration from the setup directory"
+	echo "diagnostics       check your install using diagnostics"
 	;;
 esac