X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/08078f621c8dbc1ecbd6e030bb0fac848cb6a01c..HEAD:/tools/salt-install/installer.sh diff --git a/tools/salt-install/installer.sh b/tools/salt-install/installer.sh index 0cb4b6e09e..36e87cca91 100755 --- a/tools/salt-install/installer.sh +++ b/tools/salt-install/installer.sh @@ -35,6 +35,11 @@ declare DOMAIN # This will be populated by loadconfig() declare -A NODES +# A bash associative array listing each role and mapping to the nodes +# that should be provisioned with this role. +# This will be populated by loadconfig() +declare -A ROLE2NODES + # The ssh user we'll use # This will be populated by loadconfig() declare DEPLOY_USER @@ -43,280 +48,439 @@ declare DEPLOY_USER # This will be populated by loadconfig() declare GITTARGET +# The public host used as an SSH jump host +# This will be populated by loadconfig() +declare USE_SSH_JUMPHOST + +# The temp file that will get used to disable envvar forwarding to avoid locale +# issues in Debian distros. +# This will be populated by loadconfig() +declare SSH_CONFFILE + checktools() { - local MISSING='' - for a in git ip ; do - if ! which $a ; then - MISSING="$MISSING $a" - fi - done - if [[ -n "$MISSING" ]] ; then - echo "Some tools are missing, please make sure you have the 'git' and 'iproute2' packages installed" - exit 1 + local MISSING='' + for a in git ip; do + if ! which $a; then + MISSING="$MISSING $a" fi + done + if [[ -n "$MISSING" ]]; then + echo "Some tools are missing, please make sure you have the 'git' and 'iproute2' packages installed" + exit 1 + fi +} + +cleanup() { + local NODE=$1 + local SSH=$(ssh_cmd "$NODE") + # Delete the old repository + $SSH $DEPLOY_USER@$NODE rm -rf ${GITTARGET}.git ${GITTARGET} } sync() { - local NODE=$1 - local BRANCH=$2 - - # Synchronizes the configuration by creating a git repository on - # each node, pushing our branch, and updating the checkout. - - if [[ "$NODE" != localhost ]] ; then - if ! ssh $DEPLOY_USER@$NODE test -d ${GITTARGET}.git ; then - - # Initialize the git repository (1st time case). We're - # actually going to make two repositories here because git - # will complain if you try to push to a repository with a - # checkout. So we're going to create a "bare" repository - # and then clone a regular repository (with a checkout) - # from that. - - ssh $DEPLOY_USER@$NODE git init --bare ${GITTARGET}.git - if ! git remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git ; then - git remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git - fi - git push $NODE $BRANCH - ssh $DEPLOY_USER@$NODE git clone ${GITTARGET}.git ${GITTARGET} - fi - - # The update case. - # - # Push to the bare repository on the remote node, then in the - # remote node repository with the checkout, pull the branch - # from the bare repository. - - git push $NODE $BRANCH - ssh $DEPLOY_USER@$NODE "git -C ${GITTARGET} checkout ${BRANCH} && git -C ${GITTARGET} pull" + local NODE=$1 + local BRANCH=$2 + + # Synchronizes the configuration by creating a git repository on + # each node, pushing our branch, and updating the checkout. + + if [[ "$NODE" != localhost ]]; then + SSH=$(ssh_cmd "$NODE") + GIT="eval $(git_cmd $NODE)" + + cleanup $NODE + + # Update the git remote for the remote repository. + if ! $GIT remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git; then + $GIT remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git fi + + # Initialize the git repository. We're + # actually going to make two repositories here because git + # will complain if you try to push to a repository with a + # checkout. So we're going to create a "bare" repository + # and then clone a regular repository (with a checkout) + # from that. + + $SSH $DEPLOY_USER@$NODE git init --bare --shared=0600 ${GITTARGET}.git + if [[ "$BRANCH" == "HEAD" ]]; then + # When deploying from an individual commit instead of a branch. This can + # happen when deploying from a Jenkins pipeline. + $GIT push $NODE HEAD:refs/heads/HEAD + $SSH $DEPLOY_USER@$NODE "umask 0077 && git clone -s ${GITTARGET}.git ${GITTARGET} && git -C ${GITTARGET} checkout remotes/origin/HEAD" + else + $GIT push $NODE $BRANCH + $SSH $DEPLOY_USER@$NODE "umask 0077 && git clone -s ${GITTARGET}.git ${GITTARGET} && git -C ${GITTARGET} checkout ${BRANCH}" + fi + fi } deploynode() { - local NODE=$1 - local ROLES=$2 + local NODE=$1 + local ROLES=$2 + local BRANCH=$3 - # Deploy a node. This runs the provision script on the node, with - # the appropriate roles. + # Deploy a node. This runs the provision script on the node, with + # the appropriate roles. - if [[ -z "$ROLES" ]] ; then - echo "No roles specified for $NODE, will deploy all roles" - else - ROLES="--roles ${ROLES}" - fi + sync $NODE $BRANCH - logfile=deploy-${NODE}-$(date -Iseconds).log + if [[ -z "$ROLES" ]]; then + echo "No roles specified for $NODE, will deploy all roles" + else + ROLES="--roles ${ROLES}" + fi - if [[ "$NODE" = localhost ]] ; then - SUDO='' - if [[ $(whoami) != 'root' ]] ; then - SUDO=sudo - fi - $SUDO ./provision.sh --config ${CONFIG_FILE} ${ROLES} 2>&1 | tee $logfile - else - ssh $DEPLOY_USER@$NODE "cd ${GITTARGET} && sudo ./provision.sh --config ${CONFIG_FILE} ${ROLES}" 2>&1 | tee $logfile + logfile=deploy-${NODE}-$(date -Iseconds).log + SSH=$(ssh_cmd "$NODE") + + if [[ "$NODE" = localhost ]]; then + SUDO='' + if [[ $(whoami) != 'root' ]]; then + SUDO=sudo fi + $SUDO ./provision.sh --config ${CONFIG_FILE} ${ROLES} 2>&1 | tee $logfile + else + $SSH $DEPLOY_USER@$NODE "cd ${GITTARGET} && git log -n1 HEAD && DISABLED_CONTROLLER=\"$DISABLED_CONTROLLER\" sudo --preserve-env=DISABLED_CONTROLLER ./provision.sh --config ${CONFIG_FILE} ${ROLES}" 2>&1 | tee $logfile + cleanup $NODE + fi +} + +checkcert() { + local CERTNAME=$1 + local CERTPATH="${CONFIG_DIR}/certs/${CERTNAME}" + if [[ ! -f "${CERTPATH}.crt" || ! -e "${CERTPATH}.key" ]]; then + echo "Missing ${CERTPATH}.crt or ${CERTPATH}.key files" + exit 1 + fi } loadconfig() { - if [[ ! -s $CONFIG_FILE ]] ; then - echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?" - fi - source ${CONFIG_FILE} - GITTARGET=arvados-deploy-config-${CLUSTER} + if ! [[ -s ${CONFIG_FILE} && -s ${CONFIG_FILE}.secrets ]]; then + echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?" + fi + source common.sh + GITTARGET=arvados-deploy-config-${CLUSTER} + + # Set up SSH so that it doesn't forward any environment variable. This is to avoid + # getting "setlocale" errors on the first run, depending on the distro being used + # to run the installer (like Debian). + SSH_CONFFILE=$(mktemp) + echo "Include config SendEnv -*" >${SSH_CONFFILE} +} + +ssh_cmd() { + local NODE=$1 + if [ -z "${USE_SSH_JUMPHOST}" -o "${NODE}" == "${USE_SSH_JUMPHOST}" -o "${NODE}" == "localhost" ]; then + echo "ssh -F ${SSH_CONFFILE}" + else + echo "ssh -F ${SSH_CONFFILE} -J ${DEPLOY_USER}@${USE_SSH_JUMPHOST}" + fi +} + +git_cmd() { + local NODE=$1 + echo "GIT_SSH_COMMAND=\"$(ssh_cmd ${NODE})\" git" } set +u subcmd="$1" set -u -if [[ -n "$subcmd" ]] ; then - shift +if [[ -n "$subcmd" ]]; then + shift fi case "$subcmd" in - initialize) - if [[ ! -f provision.sh ]] ; then - echo "Must be run from arvados/tools/salt-install" - exit - fi - - checktools - - set +u - SETUPDIR=$1 - PARAMS=$2 - SLS=$3 - TERRAFORM=$4 - set -u - - err= - if [[ -z "$PARAMS" || ! -f local.params.example.$PARAMS ]] ; then - echo "Not found: local.params.example.$PARAMS" - echo "Expected one of multiple_hosts, single_host_multiple_hostnames, single_host_single_hostname" - err=1 - fi - - if [[ -z "$SLS" || ! -d config_examples/$SLS ]] ; then - echo "Not found: config_examples/$SLS" - echo "Expected one of multi_host/aws, single_host/multiple_hostnames, single_host/single_hostname" - err=1 - fi - - if [[ -z "$SETUPDIR" || -z "$PARAMS" || -z "$SLS" ]]; then - echo "installer.sh " - err=1 - fi - - if [[ -n "$err" ]] ; then - exit 1 - fi - - echo "Initializing $SETUPDIR" - git init $SETUPDIR - cp -r *.sh tests $SETUPDIR - - cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE} - cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR} - - if [[ -n "$TERRAFORM" ]] ; then - mkdir $SETUPDIR/terraform - cp -r $TERRAFORM/* $SETUPDIR/terraform/ - fi - - cd $SETUPDIR - echo '*.log' > .gitignore - - git add *.sh ${CONFIG_FILE} ${CONFIG_DIR} tests .gitignore - git commit -m"initial commit" - - echo - echo "Setup directory $SETUPDIR initialized." - if [[ -n "$TERRAFORM" ]] ; then - (cd $SETUPDIR/terraform/vpc && terraform init) - (cd $SETUPDIR/terraform/data-storage && terraform init) - (cd $SETUPDIR/terraform/services && terraform init) - echo "Now go to $SETUPDIR, customize 'terraform/vpc/terraform.tfvars' as needed, then run 'installer.sh terraform'" - else - echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'" - fi - ;; - - terraform) - logfile=terraform-$(date -Iseconds).log - (cd terraform/vpc && terraform apply) 2>&1 | tee -a $logfile - (cd terraform/data-storage && terraform apply) 2>&1 | tee -a $logfile - (cd terraform/services && terraform apply) 2>&1 | grep -v letsencrypt_iam_secret_access_key | tee -a $logfile - (cd terraform/services && echo -n 'letsencrypt_iam_secret_access_key = ' && terraform output letsencrypt_iam_secret_access_key) 2>&1 | tee -a $logfile - ;; - - generate-tokens) - for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do - echo ${i}=$(tr -dc A-Za-z0-9 " + err=1 + fi + + if [[ -n "$err" ]]; then + exit 1 + fi + + echo "Initializing $SETUPDIR" + git init --shared=0600 $SETUPDIR + cp -r *.sh tests $SETUPDIR + + cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE} + cp local.params.secrets.example $SETUPDIR/${CONFIG_FILE}.secrets + cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR} + + if [[ -n "$TERRAFORM" ]]; then + mkdir $SETUPDIR/terraform + cp -r $TERRAFORM/* $SETUPDIR/terraform/ + fi + + cd $SETUPDIR + echo '*.log' >.gitignore + echo '**/.terraform' >>.gitignore + echo '**/.infracost' >>.gitignore + + if [[ -n "$TERRAFORM" ]]; then + git add terraform + fi + + git add *.sh ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} tests .gitignore + git commit -m"initial commit" + + echo + echo "Setup directory $SETUPDIR initialized." + if [[ -n "$TERRAFORM" ]]; then + (cd $SETUPDIR/terraform/vpc && terraform init) + (cd $SETUPDIR/terraform/data-storage && terraform init) + (cd $SETUPDIR/terraform/services && terraform init) + echo "Now go to $SETUPDIR, customize 'terraform/vpc/terraform.tfvars' as needed, then run 'installer.sh terraform'" + else + echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}', '${CONFIG_FILE}.secrets' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'" + fi + ;; + +terraform) + logfile=terraform-$(date -Iseconds).log + (cd terraform/vpc && terraform apply -auto-approve) 2>&1 | tee -a $logfile + (cd terraform/data-storage && terraform apply -auto-approve) 2>&1 | tee -a $logfile + (cd terraform/services && terraform apply -auto-approve) 2>&1 | grep -v letsencrypt_iam_secret_access_key | tee -a $logfile + (cd terraform/services && echo -n 'letsencrypt_iam_secret_access_key = ' && terraform output letsencrypt_iam_secret_access_key) 2>&1 | tee -a $logfile + ;; + +terraform-destroy) + logfile=terraform-$(date -Iseconds).log + (cd terraform/services && terraform destroy) 2>&1 | tee -a $logfile + (cd terraform/data-storage && terraform destroy) 2>&1 | tee -a $logfile + (cd terraform/vpc && terraform destroy) 2>&1 | tee -a $logfile + ;; + +generate-tokens) + for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN DATABASE_PASSWORD; do + echo ${i}=$( + tr -dc A-Za-z0-9