From 2b541090d500166243c84a9d7d2f9dbedf9f7360 Mon Sep 17 00:00:00 2001 From: Lucas Di Pentima Date: Wed, 21 Jun 2023 16:11:11 -0300 Subject: [PATCH] 20665: Handles secrets on a separate file. This allows the admin to give special treatment to it, to comply with security policies that the local organization may have. Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima --- .../multi_host/aws/pillars/arvados.sls | 8 ++----- tools/salt-install/installer.sh | 12 ++++++---- .../local.params.example.multiple_hosts | 11 --------- ...ams.example.single_host_multiple_hostnames | 9 ------- ...params.example.single_host_single_hostname | 9 ------- .../salt-install/local.params.secrets.example | 24 +++++++++++++++++++ tools/salt-install/provision.sh | 15 +++++++----- 7 files changed, 42 insertions(+), 46 deletions(-) create mode 100644 tools/salt-install/local.params.secrets.example diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls index 51842b6e2a..98fcf5f6d9 100644 --- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls @@ -115,6 +115,7 @@ arvados: {%- endif %} ### CONTAINERS + {%- set dispatcher_ssh_privkey = "__DISPATCHER_SSH_PRIVKEY__" %} Containers: MaxRetryAttempts: 10 CloudVMs: @@ -131,12 +132,7 @@ arvados: SecurityGroupIDs: ['sg-FIXMEFIXMEFIXMEFI'] SubnetID: subnet-FIXMEFIXMEFIXMEFI IAMInstanceProfile: __CLUSTER__-compute-node-00-iam-role - DispatchPrivateKey: | - -----BEGIN OPENSSH PRIVATE KEY----- - Read https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html#sshkeypair - for details on how to create this key. - FIXMEFIXMEFIXME replace this with your dispatcher ssh private key - -----END OPENSSH PRIVATE KEY----- + DispatchPrivateKey: {{ dispatcher_ssh_privkey|yaml_dquote }} ### VOLUMES ## This should usually match all your `keepstore` instances diff --git a/tools/salt-install/installer.sh b/tools/salt-install/installer.sh index c86e9b02c3..55cbd36d94 100755 --- a/tools/salt-install/installer.sh +++ b/tools/salt-install/installer.sh @@ -127,9 +127,10 @@ deploynode() { } loadconfig() { - if [[ ! -s $CONFIG_FILE ]] ; then + if [ ! -s ${CONFIG_FILE} -o ! -s ${CONFIG_FILE}.secrets ]; then echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?" fi + source ${CONFIG_FILE}.secrets source ${CONFIG_FILE} GITTARGET=arvados-deploy-config-${CLUSTER} } @@ -198,6 +199,7 @@ case "$subcmd" in cp -r *.sh tests $SETUPDIR cp local.params.example.$PARAMS $SETUPDIR/${CONFIG_FILE} + cp local.params.secrets.example $SETUPDIR/${CONFIG_FILE}.secrets cp -r config_examples/$SLS $SETUPDIR/${CONFIG_DIR} if [[ -n "$TERRAFORM" ]] ; then @@ -214,7 +216,7 @@ case "$subcmd" in git add terraform fi - git add *.sh ${CONFIG_FILE} ${CONFIG_DIR} tests .gitignore + git add *.sh ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} tests .gitignore git commit -m"initial commit" echo @@ -225,7 +227,7 @@ case "$subcmd" in (cd $SETUPDIR/terraform/services && terraform init) echo "Now go to $SETUPDIR, customize 'terraform/vpc/terraform.tfvars' as needed, then run 'installer.sh terraform'" else - echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'" + echo "Now go to $SETUPDIR, customize '${CONFIG_FILE}', '${CONFIG_FILE}.secrets' and '${CONFIG_DIR}' as needed, then run 'installer.sh deploy'" fi ;; @@ -259,7 +261,7 @@ case "$subcmd" in loadconfig - if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_DIR} ; then + if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_FILE}.secrets ${CONFIG_DIR} ; then echo echo "Some parameters still need to be updated. Please fix them and then re-run deploy." exit 1 @@ -270,7 +272,7 @@ case "$subcmd" in set -x git add -A - if ! git diff --cached --exit-code ; then + if ! git diff --cached --exit-code --quiet ; then git commit -m"prepare for deploy" fi diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts index 463ee4c101..d1cdfeb3c6 100644 --- a/tools/salt-install/local.params.example.multiple_hosts +++ b/tools/salt-install/local.params.example.multiple_hosts @@ -21,7 +21,6 @@ INITIAL_USER=admin # If not specified, the initial user email will be composed as # INITIAL_USER@DOMAIN INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work" -INITIAL_USER_PASSWORD="fixmepassword" # Use a public node as a jump host for SSH sessions. This allows running the # installer from the outside of the cluster's local network and still reach @@ -29,14 +28,6 @@ INITIAL_USER_PASSWORD="fixmepassword" # Comment out to disable. USE_SSH_JUMPHOST="controller.${DOMAIN}" -# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS -BLOB_SIGNING_KEY=fixmeblobsigningkeymushaveatleast32characters -MANAGEMENT_TOKEN=fixmemanagementtokenmushaveatleast32characters -SYSTEM_ROOT_TOKEN=fixmesystemroottokenmushaveatleast32characters -ANONYMOUS_USER_TOKEN=fixmeanonymoususertokenmushaveatleast32characters -WORKBENCH_SECRET_KEY=fixmeworkbenchsecretkeymushaveatleast32characters -DATABASE_PASSWORD=fixmeplease_set_this_to_some_secure_value - # SSL CERTIFICATES # Arvados requires SSL certificates to work correctly. This installer supports these options: # * self-signed: let the installer create self-signed certificate(s) @@ -52,8 +43,6 @@ USE_LETSENCRYPT_ROUTE53="yes" # RRs in the route53 zone for the cluster. # WARNING!: If AWS credentials files already exist in the hosts, they won't be replaced. LE_AWS_REGION="us-east-1" -LE_AWS_ACCESS_KEY_ID="AKIABCDEFGHIJKLMNOPQ" -LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey" # If you going to provide your own certificates for Arvados, the provision script can # help you deploy them. In order to do that, you need to set `SSL_MODE=bring-your-own` above, diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames index 5633c6cbf0..5b3135287f 100644 --- a/tools/salt-install/local.params.example.single_host_multiple_hostnames +++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames @@ -20,15 +20,6 @@ INITIAL_USER=admin # If not specified, the initial user email will be composed as # INITIAL_USER@CLUSTER.DOMAIN INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work" -INITIAL_USER_PASSWORD="fixmepassword" - -# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS -BLOB_SIGNING_KEY=fixmeblobsigningkeymushaveatleast32characters -MANAGEMENT_TOKEN=fixmemanagementtokenmushaveatleast32characters -SYSTEM_ROOT_TOKEN=fixmesystemroottokenmushaveatleast32characters -ANONYMOUS_USER_TOKEN=fixmeanonymoususertokenmushaveatleast32characters -WORKBENCH_SECRET_KEY=fixmeworkbenchsecretkeymushaveatleast32characters -DATABASE_PASSWORD=fixmeplease_set_this_to_some_secure_value # SSL CERTIFICATES # Arvados requires SSL certificates to work correctly. This installer supports these options: diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname index 0c4f5c3567..7af8898a50 100644 --- a/tools/salt-install/local.params.example.single_host_single_hostname +++ b/tools/salt-install/local.params.example.single_host_single_hostname @@ -20,15 +20,6 @@ INITIAL_USER=admin # If not specified, the initial user email will be composed as # INITIAL_USER@CLUSTER.DOMAIN INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work" -INITIAL_USER_PASSWORD="fixmepassword" - -# Populate these values with random strings -BLOB_SIGNING_KEY=fixmeblobsigningkeymushaveatleast32characters -MANAGEMENT_TOKEN=fixmemanagementtokenmushaveatleast32characters -SYSTEM_ROOT_TOKEN=fixmesystemroottokenmushaveatleast32characters -ANONYMOUS_USER_TOKEN=fixmeanonymoususertokenmushaveatleast32characters -WORKBENCH_SECRET_KEY=fixmeworkbenchsecretkeymushaveatleast32characters -DATABASE_PASSWORD=fixmeplease_set_this_to_some_secure_value # SSL CERTIFICATES # Arvados requires SSL certificates to work correctly. This installer supports these options: diff --git a/tools/salt-install/local.params.secrets.example b/tools/salt-install/local.params.secrets.example new file mode 100644 index 0000000000..bec56e00be --- /dev/null +++ b/tools/salt-install/local.params.secrets.example @@ -0,0 +1,24 @@ +########################################################## +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: CC-BY-SA-3.0 + +# These are the security-sensitive parameters to configure the installation + +INITIAL_USER_PASSWORD="fixme" + +# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS +BLOB_SIGNING_KEY=fixmeblobsigningkeymushaveatleast32characters +MANAGEMENT_TOKEN=fixmemanagementtokenmushaveatleast32characters +SYSTEM_ROOT_TOKEN=fixmesystemroottokenmushaveatleast32characters +ANONYMOUS_USER_TOKEN=fixmeanonymoususertokenmushaveatleast32characters +WORKBENCH_SECRET_KEY=fixmeworkbenchsecretkeymushaveatleast32characters +DATABASE_PASSWORD=fixmeplease_set_this_to_some_secure_value + +LE_AWS_ACCESS_KEY_ID="FIXME" +LE_AWS_SECRET_ACCESS_KEY="fixme" + +# Read https://doc.arvados.org/install/crunch2-cloud/install-compute-node.html#sshkeypair +# for details on how to create this key. +DISPATCHER_SSH_PRIVKEY="fixme" + diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh index f90386652b..e438dfa775 100755 --- a/tools/salt-install/provision.sh +++ b/tools/salt-install/provision.sh @@ -241,11 +241,12 @@ arguments ${@} declare -A NODES -if [ -s ${CONFIG_FILE} ]; then +if [ -s ${CONFIG_FILE} -a -s ${CONFIG_FILE}.secrets ]; then + source ${CONFIG_FILE}.secrets source ${CONFIG_FILE} else echo >&2 "You don't seem to have a config file with initial values." - echo >&2 "Please create a '${CONFIG_FILE}' file as described in" + echo >&2 "Please create a '${CONFIG_FILE}' & '${CONFIG_FILE}.secrets' files as described in" echo >&2 " * https://doc.arvados.org/install/salt-single-host.html#single_host, or" echo >&2 " * https://doc.arvados.org/install/salt-multi-host.html#multi_host_multi_hostnames" exit 1 @@ -259,8 +260,8 @@ if [ ! -d ${CONFIG_DIR} ]; then exit 1 fi -if grep -rni 'fixme' ${CONFIG_FILE} ${CONFIG_DIR} ; then - echo >&2 "The config file ${CONFIG_FILE} has some parameters that need to be modified." +if grep -rni 'fixme' ${CONFIG_FILE}.secrets ${CONFIG_FILE} ${CONFIG_DIR} ; then + echo >&2 "The config files has some parameters that need to be modified." echo >&2 "Please, fix them and re-run the provision script." exit 1 fi @@ -459,7 +460,8 @@ for f in $(ls "${SOURCE_PILLARS_DIR}"/*); do s#__CONTROLLER_MAX_CONCURRENT_REQUESTS__#${CONTROLLER_MAX_CONCURRENT_REQUESTS}#g; s#__MONITORING_USERNAME__#${MONITORING_USERNAME}#g; s#__MONITORING_EMAIL__#${MONITORING_EMAIL}#g; - s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g" \ + s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g; + s#__DISPATCHER_SSH_PRIVKEY__#${DISPATCHER_SSH_PRIVKEY//$'\n'/\\n}#g" \ "${f}" > "${P_DIR}"/$(basename "${f}") done @@ -538,7 +540,8 @@ if [ -d "${SOURCE_STATES_DIR}" ]; then s#__CONTROLLER_MAX_CONCURRENT_REQUESTS__#${CONTROLLER_MAX_CONCURRENT_REQUESTS}#g; s#__MONITORING_USERNAME__#${MONITORING_USERNAME}#g; s#__MONITORING_EMAIL__#${MONITORING_EMAIL}#g; - s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g" \ + s#__MONITORING_PASSWORD__#${MONITORING_PASSWORD}#g; + s#__DISPATCHER_SSH_PRIVKEY__#${DISPATCHER_SSH_PRIVKEY//$'\n'/\\n}#g" \ "${f}" > "${F_DIR}/extra/extra"/$(basename "${f}") done fi -- 2.30.2