X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/02f2c38ee8f2a5d2e33a7bd2ff033303e6861aaf..HEAD:/tools/salt-install/installer.sh diff --git a/tools/salt-install/installer.sh b/tools/salt-install/installer.sh index 91ade4212d..2a0460c648 100755 --- a/tools/salt-install/installer.sh +++ b/tools/salt-install/installer.sh @@ -14,6 +14,7 @@ # set -eu +set -o pipefail # The parameter file declare CONFIG_FILE=local.params @@ -34,6 +35,11 @@ declare DOMAIN # This will be populated by loadconfig() declare -A NODES +# A bash associative array listing each role and mapping to the nodes +# that should be provisioned with this role. +# This will be populated by loadconfig() +declare -A ROLE2NODES + # The ssh user we'll use # This will be populated by loadconfig() declare DEPLOY_USER @@ -42,220 +48,447 @@ declare DEPLOY_USER # This will be populated by loadconfig() declare GITTARGET +# The public host used as an SSH jump host +# This will be populated by loadconfig() +declare USE_SSH_JUMPHOST + +# The temp file that will get used to disable envvar forwarding to avoid locale +# issues in Debian distros. +# This will be populated by loadconfig() +declare SSH_CONFFILE + +checktools() { + local MISSING='' + for a in git ip; do + if ! which $a; then + MISSING="$MISSING $a" + fi + done + if [[ -n "$MISSING" ]]; then + echo "Some tools are missing, please make sure you have the 'git' and 'iproute2' packages installed" + exit 1 + fi +} + +cleanup() { + local NODE=$1 + local SSH=$(ssh_cmd "$NODE") + # Delete the old repository + $SSH $DEPLOY_USER@$NODE rm -rf ${GITTARGET}.git ${GITTARGET} +} + sync() { - local NODE=$1 - local BRANCH=$2 - - # Synchronizes the configuration by creating a git repository on - # each node, pushing our branch, and updating the checkout. - - if [[ "$NODE" != localhost ]] ; then - if ! ssh $NODE test -d ${GITTARGET}.git ; then - - # Initialize the git repository (1st time case). We're - # actually going to make two repositories here because git - # will complain if you try to push to a repository with a - # checkout. So we're going to create a "bare" repository - # and then clone a regular repository (with a checkout) - # from that. - - ssh $NODE git init --bare ${GITTARGET}.git - if ! git remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git ; then - git remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git - fi - git push $NODE $BRANCH - ssh $NODE git clone ${GITTARGET}.git ${GITTARGET} - fi - - # The update case. - # - # Push to the bare repository on the remote node, then in the - # remote node repository with the checkout, pull the branch - # from the bare repository. - - git push $NODE $BRANCH - ssh $NODE "git -C ${GITTARGET} checkout ${BRANCH} && git -C ${GITTARGET} pull" + local NODE=$1 + local BRANCH=$2 + + # Synchronizes the configuration by creating a git repository on + # each node, pushing our branch, and updating the checkout. + + if [[ "$NODE" != localhost ]]; then + SSH=$(ssh_cmd "$NODE") + GIT="eval $(git_cmd $NODE)" + + cleanup $NODE + + # Update the git remote for the remote repository. + if ! $GIT remote add $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git; then + $GIT remote set-url $NODE $DEPLOY_USER@$NODE:${GITTARGET}.git fi + + # Initialize the git repository. We're + # actually going to make two repositories here because git + # will complain if you try to push to a repository with a + # checkout. So we're going to create a "bare" repository + # and then clone a regular repository (with a checkout) + # from that. + + $SSH $DEPLOY_USER@$NODE git init --bare --shared=0600 ${GITTARGET}.git + if [[ "$BRANCH" == "HEAD" ]]; then + # When deploying from an individual commit instead of a branch. This can + # happen when deploying from a Jenkins pipeline. + $GIT push $NODE HEAD:refs/heads/HEAD + $SSH $DEPLOY_USER@$NODE "umask 0077 && git clone -s ${GITTARGET}.git ${GITTARGET} && git -C ${GITTARGET} checkout remotes/origin/HEAD" + else + $GIT push $NODE $BRANCH + $SSH $DEPLOY_USER@$NODE "umask 0077 && git clone -s ${GITTARGET}.git ${GITTARGET} && git -C ${GITTARGET} checkout ${BRANCH}" + fi + fi } deploynode() { - local NODE=$1 - local ROLES=$2 + local NODE=$1 + local ROLES=$2 + local BRANCH=$3 - # Deploy a node. This runs the provision script on the node, with - # the appropriate roles. + # Deploy a node. This runs the provision script on the node, with + # the appropriate roles. - if [[ -z "$ROLES" ]] ; then - echo "No roles declared for '$NODE' in ${CONFIG_FILE}" - exit 1 - fi + sync $NODE $BRANCH - logfile=deploy-${NODE}-$(date -Iseconds).log + if [[ -z "$ROLES" ]]; then + echo "No roles specified for $NODE, will deploy all roles" + else + ROLES="--roles ${ROLES}" + fi - if [[ "$NODE" = localhost ]] ; then - sudo ./provision.sh --config ${CONFIG_FILE} --roles ${ROLES} 2>&1 | tee $logfile - else - ssh $DEPLOY_USER@$NODE "cd ${GITTARGET} && sudo ./provision.sh --config ${CONFIG_FILE} --roles ${ROLES}" 2>&1 | tee $logfile + logfile=deploy-${NODE}-$(date -Iseconds).log + SSH=$(ssh_cmd "$NODE") + + if [[ "$NODE" = localhost ]]; then + SUDO='' + if [[ $(whoami) != 'root' ]]; then + SUDO=sudo fi + $SUDO ./provision.sh --config ${CONFIG_FILE} ${ROLES} 2>&1 | tee $logfile + else + $SSH $DEPLOY_USER@$NODE "cd ${GITTARGET} && git log -n1 HEAD && DISABLED_CONTROLLER=\"$DISABLED_CONTROLLER\" sudo --preserve-env=DISABLED_CONTROLLER ./provision.sh --config ${CONFIG_FILE} ${ROLES}" 2>&1 | tee $logfile + cleanup $NODE + fi +} + +checkcert() { + local CERTNAME=$1 + local CERTPATH="${CONFIG_DIR}/certs/${CERTNAME}" + if [[ ! -f "${CERTPATH}.crt" || ! -e "${CERTPATH}.key" ]]; then + echo "Missing ${CERTPATH}.crt or ${CERTPATH}.key files" + exit 1 + fi } loadconfig() { - if [[ ! -s $CONFIG_FILE ]] ; then - echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?" - fi - source ${CONFIG_FILE} - GITTARGET=arvados-deploy-config-${CLUSTER} + if ! [[ -s ${CONFIG_FILE} && -s ${CONFIG_FILE}.secrets ]]; then + echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?" + fi + source common.sh + GITTARGET=arvados-deploy-config-${CLUSTER} + + # Set up SSH so that it doesn't forward any environment variable. This is to avoid + # getting "setlocale" errors on the first run, depending on the distro being used + # to run the installer (like Debian). + SSH_CONFFILE=$(mktemp) + echo "Include config SendEnv -*" >${SSH_CONFFILE} +} + +ssh_cmd() { + local NODE=$1 + if [ -z "${USE_SSH_JUMPHOST}" -o "${NODE}" == "${USE_SSH_JUMPHOST}" -o "${NODE}" == "localhost" ]; then + echo "ssh -F ${SSH_CONFFILE}" + else + echo "ssh -F ${SSH_CONFFILE} -J ${DEPLOY_USER}@${USE_SSH_JUMPHOST}" + fi +} + +git_cmd() { + local NODE=$1 + echo "GIT_SSH_COMMAND=\"$(ssh_cmd ${NODE})\" git" } +set +u subcmd="$1" -if [[ -n "$subcmd" ]] ; then - shift +set -u + +if [[ -n "$subcmd" ]]; then + shift fi case "$subcmd" in - initialize) - if [[ ! -f provision.sh ]] ; then - echo "Must be run from arvados/tools/salt-install" - exit - fi - - set +u - SETUPDIR=$1 - PARAMS=$2 - SLS=$3 - set -u - - err= - if [[ -z "$PARAMS" || ! -f local.params.example.$PARAMS ]] ; then - echo "Not found: local.params.example.$PARAMS" - echo "Expected one of multiple_hosts, single_host_multiple_hostnames, single_host_single_hostname" - err=1 - fi - - if [[ -z "$SLS" || ! -d config_examples/$SLS ]] ; then - echo "Not found: config_examples/$SLS" - echo "Expected one of multi_host/aws, single_host/multiple_hostnames, single_host/single_hostname" - err=1 - fi - - if [[ -z "$SETUPDIR" || -z "$PARAMS" || -z "$SLS" ]]; then - echo "installer.sh " - err=1 - fi - - if [[ -n "$err" ]] ; then - exit 1 - fi - - echo "Initializing $SETUPDIR" - git init $SETUPDIR - cp -r *.sh tests $SETUPDIR - - cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE} - cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR} - - cd $SETUPDIR - echo '*.log' > .gitignore - - git add *.sh ${CONFIG_FILE} ${CONFIG_DIR} tests .gitignore - git commit -m"initial commit" - - echo "setup directory initialized, now go to $SETUPDIR, edit '${CONFIG_FILE}' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'" - ;; - deploy) - set +u - NODE=$1 - set -u - - loadconfig - - if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_DIR} ; then - echo - echo "Some parameters still need to be updated. Please fix them and then re-run deploy." - exit 1 - fi - - BRANCH=$(git branch --show-current) - - set -x - - git add -A - if ! git diff --cached --exit-code ; then - git commit -m"prepare for deploy" - fi - - if [[ -z "$NODE" ]]; then - for NODE in "${!NODES[@]}" - do - # First, push the git repo to each node. This also - # confirms that we have git and can log into each - # node. - sync $NODE $BRANCH - done - - for NODE in "${!NODES[@]}" - do - # Do 'database' role first, - if [[ "${NODES[$NODE]}" =~ database ]] ; then - deploynode $NODE ${NODES[$NODE]} - unset NODES[$NODE] - fi - done - - for NODE in "${!NODES[@]}" - do - # then 'api' or 'controller' roles - if [[ "${NODES[$NODE]}" =~ (api|controller) ]] ; then - deploynode $NODE ${NODES[$NODE]} - unset NODES[$NODE] - fi - done - - for NODE in "${!NODES[@]}" - do - # Everything else (we removed the nodes that we - # already deployed from the list) - deploynode $NODE ${NODES[$NODE]} - done - else - # Just deploy the node that was supplied on the command line. - sync $NODE $BRANCH - deploynode $NODE - fi - - echo - echo "Completed deploy, run 'installer.sh diagnostics' to verify the install" - - ;; - diagnostics) - loadconfig - - set +u - declare LOCATION=$1 - set -u - - if ! which arvados-client ; then - echo "arvados-client not found, install 'arvados-client' package with 'apt-get' or 'yum'" - exit 1 - fi - - if [[ -z "$LOCATION" ]] ; then - echo "Need to provide '-internal-client' or '-external-client'" - echo - echo "-internal-client You are running this on the same private network as the Arvados cluster (e.g. on one of the Arvados nodes)" - echo "-external-client You are running this outside the private network of the Arvados cluster (e.g. your workstation)" - exit 1 - fi - - export ARVADOS_API_HOST="${CLUSTER}.${DOMAIN}" - export ARVADOS_API_TOKEN="$SYSTEM_ROOT_TOKEN" - - arvados-client diagnostics $LOCATION - ;; - *) - echo "Arvados installer" - echo "" - echo "initialize initialize the setup directory for configuration" - echo "deploy deploy the configuration from the setup directory" - echo "diagnostics check your install using diagnostics" - ;; +initialize) + if [[ ! -f provision.sh ]]; then + echo "Must be run from arvados/tools/salt-install" + exit + fi + + checktools + + set +u + SETUPDIR=$1 + PARAMS=$2 + SLS=$3 + TERRAFORM=$4 + set -u + + err= + if [[ -z "$PARAMS" || ! -f local.params.example.$PARAMS ]]; then + echo "Not found: local.params.example.$PARAMS" + echo "Expected one of multiple_hosts, single_host_multiple_hostnames, single_host_single_hostname" + err=1 + fi + + if [[ -z "$SLS" || ! -d config_examples/$SLS ]]; then + echo "Not found: config_examples/$SLS" + echo "Expected one of multi_host/aws, single_host/multiple_hostnames, single_host/single_hostname" + err=1 + fi + + if [[ -z "$SETUPDIR" || -z "$PARAMS" || -z "$SLS" ]]; then + echo "installer.sh " + err=1 + fi + + if [[ -n "$err" ]]; then + exit 1 + fi + + echo "Initializing $SETUPDIR" + git init --shared=0600 $SETUPDIR + cp -r *.sh tests $SETUPDIR + + cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE} + cp local.params.secrets.example $SETUPDIR/${CONFIG_FILE}.secrets + cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR} + + if [[ -n "$TERRAFORM" ]]; then + mkdir $SETUPDIR/terraform + cp -r $TERRAFORM/* $SETUPDIR/terraform/ + fi + + cd $SETUPDIR + echo '*.log' >.gitignore + echo '**/.terraform' >>.gitignore + echo '**/.infracost' >>.gitignore + + if [[ -n "$TERRAFORM" ]]; then + git add terraform + fi + + git add *.sh ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} tests .gitignore + git commit -m"initial commit" + + echo + echo "Setup directory $SETUPDIR initialized." + if [[ -n "$TERRAFORM" ]]; then + (cd $SETUPDIR/terraform/vpc && terraform init) + (cd $SETUPDIR/terraform/data-storage && terraform init) + (cd $SETUPDIR/terraform/services && terraform init) + echo "Now go to $SETUPDIR, customize 'terraform/vpc/terraform.tfvars' as needed, then run 'installer.sh terraform'" + else + echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}', '${CONFIG_FILE}.secrets' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'" + fi + ;; + +terraform) + logfile=terraform-$(date -Iseconds).log + (cd terraform/vpc && terraform apply -auto-approve) 2>&1 | tee -a $logfile + (cd terraform/data-storage && terraform apply -auto-approve) 2>&1 | tee -a $logfile + (cd terraform/services && \ + terraform apply -auto-approve) 2>&1 | \ + grep -v letsencrypt_iam_secret_access_key | \ + grep -v database_password | \ + tee -a $logfile + (cd terraform/services && \ + echo -n 'letsencrypt_iam_secret_access_key = ' && \ + terraform output letsencrypt_iam_secret_access_key && \ + echo -n 'database_password = ' && \ + terraform output database_password) 2>&1 | tee -a $logfile + ;; + +terraform-destroy) + logfile=terraform-$(date -Iseconds).log + (cd terraform/services && terraform destroy) 2>&1 | tee -a $logfile + (cd terraform/data-storage && terraform destroy) 2>&1 | tee -a $logfile + (cd terraform/vpc && terraform destroy) 2>&1 | tee -a $logfile + ;; + +generate-tokens) + for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN DATABASE_PASSWORD; do + echo ${i}=$( + tr -dc A-Za-z0-9