From 6fc73fc47f81b90889bcacdd54c16c7715b0abfd Mon Sep 17 00:00:00 2001
From: Ward Vandewege <ward@curii.com>
Date: Sun, 20 Feb 2022 19:44:24 -0500
Subject: [PATCH] 18791: update documentation, fix single host/single hostname
 installer        with self-signed certificates. Fix tests/run-test.sh so it
 works        even with just crunch-dispatch-local.

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>
---
 .../_install_custom_certificates.liquid       |  4 +-
 .../salt-multi-host.html.textile.liquid       |  2 +
 .../salt-single-host.html.textile.liquid      | 49 +++++++++++++++----
 doc/install/salt.html.textile.liquid          |  2 -
 .../local.params.example.multiple_hosts       | 16 +++---
 ...ams.example.single_host_multiple_hostnames | 16 +++---
 ...params.example.single_host_single_hostname | 14 ++----
 tools/salt-install/provision.sh               | 36 +++++++-------
 tools/salt-install/tests/run-test.sh          |  2 +-
 9 files changed, 83 insertions(+), 58 deletions(-)

diff --git a/doc/_includes/_install_custom_certificates.liquid b/doc/_includes/_install_custom_certificates.liquid
index 4a4aff5cfb..3c6486505c 100644
--- a/doc/_includes/_install_custom_certificates.liquid
+++ b/doc/_includes/_install_custom_certificates.liquid
@@ -4,7 +4,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-If you plan to use custom certificates, please set the variable <i>USE_LETSENCRYPT=no</i> and copy your certificates to the directory specified with the variable @CUSTOM_CERTS_DIR@ (usually "./certs") in the remote directory where you copied the @provision.sh@ script. From this dir, the provision script will install the certificates required for the role you're installing.
+If you plan to use custom certificates, please set the variable <i>SSL_MODE=bring-your-own</i> and copy your certificates to the directory specified with the variable @CUSTOM_CERTS_DIR@ (usually "./certs") in the remote directory where you copied the @provision.sh@ script. From this dir, the provision script will install the certificates required for the role you're installing.
 
 The script expects cert/key files with these basenames (matching the role except for <i>keepweb</i>, which is split in both <i>download / collections</i>):
 
@@ -24,3 +24,5 @@ Ie., for 'keepproxy', the script will look for
 ${CUSTOM_CERTS_DIR}/keepproxy.key
 </code></pre>
 </notextile>
+
+Make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
index 10f2e32ef1..f8723a0cef 100644
--- a/doc/install/salt-multi-host.html.textile.liquid
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -100,6 +100,8 @@ The <i>multi_host</i> example includes Let's Encrypt salt code to automatically
 
 {% include 'install_custom_certificates' %}
 
+If you want to use valid certificates provided by Let's Encrypt, set the variable <i>SSL_MODE=lets-encrypt</i> and make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
+
 h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
 
 You will need further customization to suit your environment, which can be done editing the Saltstack pillars and states files. Pay particular attention to the <i>pillars/arvados.sls</i> file, where you will need to provide some information that describes your environment.
diff --git a/doc/install/salt-single-host.html.textile.liquid b/doc/install/salt-single-host.html.textile.liquid
index 0f06412f91..039a4b1481 100644
--- a/doc/install/salt-single-host.html.textile.liquid
+++ b/doc/install/salt-single-host.html.textile.liquid
@@ -9,6 +9,8 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
+# "Limitations of the single host install":#limitations
+# "Prerequisites":#prerequisites
 # "Single host install using the provision.sh script":#single_host
 # "Choose the desired configuration":#choose_configuration
 ## "Single host / single hostname":#single_host_single_hostnames
@@ -22,16 +24,45 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 # "Test the installed cluster running a simple workflow":#test_install
 # "After the installation":#post_install
 
-h2(#single_host). Single host install using the provision.sh script
+h2(#limitations). Limitations of the single host install
 
-<b>NOTE: The single host installation is not recommended for production use.</b>
+<b>NOTE: The single host installation is a good choice when evaluating Arvados, but it is not recommended for production use.</b>
 
-{% include 'branchname' %}
+Using the default configuration, this installation method has a number of limitations:
+
+* all services run on the same machine, and they will compete for resources.
+* it uses the local machine disk for Keep storage (under the @/tmp@ directory).
+* it installs the @crunch-dispatch-local@ dispatcher, which can run just one concurrent CWL job. This job will be executed on the machine that runs all the Arvados services. Most workflows require at least two concurrent CWL jobs, one for the workflow runner, and one for the payload.
+
+It is possible to start with the single host installation method and modify the Arvados configuration file later to address these limitations. E.g. switch to a "different storage volume setup":{{site.baseurl}}/install/configure-s3-object-storage.html for Keep, and switch to "the cloud dispatcher":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html to provision compute resources dynamically.
+
+h2(#prerequisites). Prerequisites and planning
+
+Arvados requires SSL for (almost) all network traffic. This installation method supports the following options for the required SSL certificate(s): @self-signed@ and @bring your own certificates@.
 
-This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git .  The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
+Prerequisites
+
+* git
+* a dedicated (virtual) machine for your Arvados server with at least 2 cores and 8 GiB of RAM, running a "supported Arvados distribution":{{site.baseurl}}/install/install-manual-prerequisites.html#supportedlinux
+* ports 9443-9445, 11002, 14202, 18002, 35101 need to be reachable from your client (configurable, see below)
+* at least one DNS hostname that resolves to the IP address of your Arvados server
+* one or more SSL certificates matching the hostname(s) in use (only when using @bring your own certificates@)
+
+h2(#single_host). Single host install using the provision.sh script
+
+{% include 'branchname' %}
 
 This procedure will install all the main Arvados components to get you up and running in a single host. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host resources and its network bandwidth. As a reference, on a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
 
+This is a package-based installation method, however the installation script is currently distributed in source form via @git@:
+
+<notextile>
+<pre><code>git clone https://git.arvados.org/arvados.git
+git checkout {{ branchname }}
+cd arvados/tools/salt-install
+</code></pre>
+</notextile>
+
 The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script.  The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
 
 After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
@@ -58,10 +89,6 @@ Edit the variables in the <i>local.params</i> file. Pay attention to the <b>*_PO
 
 The <i>single_host</i> examples use self-signed SSL certificates, which are deployed using the same mechanism used to deploy custom certificates.
 
-{% include 'install_custom_certificates' %}
-
-If you want to use valid certificates provided by Let's Encrypt, please set the variable <i>USE_LETSENCRYPT=yes</i> and make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
-
 h3(#single_host_multiple_hostnames). Single host / multiple hostnames (Alternative configuration)
 <notextile>
 <pre><code>cp local.params.example.single_host_multiple_hostnames local.params
@@ -71,6 +98,8 @@ cp -r config_examples/single_host/multiple_hostnames local_config_dir
 
 Edit the variables in the <i>local.params</i> file.
 
+{% include 'install_custom_certificates' %}
+
 h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
 
 If you want or need further customization, you can edit the Saltstack pillars and states files. Pay particular attention to the <i>pillars/arvados.sls</i> one. Any extra <i>state</i> file you add under <i>local_config_dir/states</i> will be added to the salt run and applied to the host.
@@ -81,7 +110,7 @@ When you finished customizing the configuration, you are ready to copy the files
 
 <notextile>
 <pre><code>scp -r provision.sh local* tests user@host:
-# if you use custom certificates (not Let's Encrypt), make sure to copy those too:
+# if you are using bring-your-own certificates, make sure to copy those too:
 # scp -r certs user@host:
 ssh user@host sudo ./provision.sh
 </code></pre>
@@ -109,7 +138,7 @@ h2(#final_steps). Final configuration steps
 
 Once the deployment went OK, you'll need to perform a few extra steps in your local browser/host to access the cluster.
 
-h3(#ca_root_certificate). Install the CA root certificate (required in both alternatives)
+h3(#ca_root_certificate). Install the CA root certificate (SSL_MODE=self-signed only)
 
 Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
 
diff --git a/doc/install/salt.html.textile.liquid b/doc/install/salt.html.textile.liquid
index 8db0ac15e4..29a6eacf3b 100644
--- a/doc/install/salt.html.textile.liquid
+++ b/doc/install/salt.html.textile.liquid
@@ -31,8 +31,6 @@ You don't need to be running a Saltstack infrastructure to install Arvados: we w
 
 This is a package-based installation method.
 
-
-
 h2(#provisioning_arvados). Provisioning Arvados with Saltstack
 
 The "tools/salt-install":https://git.arvados.org/arvados.git/tree/{{ branchname }}:/tools/salt-install directory in the Arvados git repository contains a script that you can run in the node/s where you want to install Arvados' components (the @provision.sh@ script) and a few configuration examples for different setups, that you can use to customize your installation.
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index cb0afecc46..cf960df70b 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -58,13 +58,13 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 DATABASE_PASSWORD=please_set_this_to_some_secure_value
 
 # SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will fail
-# to communicate and can silently drop traffic. You can try to use the Letsencrypt
-# salt formula (https://github.com/saltstack-formulas/letsencrypt-formula) to try to
-# automatically obtain and install SSL certificates for your instances or set this
-# variable to "no", provide and upload your own certificates to the instances and
-# modify the 'nginx_*' salt pillars accordingly (see CUSTOM_CERTS_DIR below)
-USE_LETSENCRYPT="yes"
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
+#
+# See https://doc.arvados.org/intall/salt-multi-host.html for more information.
+SSL_MODE="lets-encrypt"
 USE_LETSENCRYPT_IAM_USER="yes"
 # For collections, we need to obtain a wildcard certificate for
 # '*.collections.<cluster>.<domain>'. This is only possible through a DNS-01 challenge.
@@ -76,7 +76,7 @@ LE_AWS_ACCESS_KEY_ID="AKIABCDEFGHIJKLMNOPQ"
 LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
 
 # If you going to provide your own certificates for Arvados, the provision script can
-# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
+# help you deploy them. In order to do that, you need to set `SSL_MODE=bring-your-own` above,
 # and copy the required certificates under the directory specified in the next line.
 # The certs will be copied from this directory by the provision script.
 # Please set it to the FULL PATH to the certs dir if you're going to use a different dir
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index ef47467e57..0fca600eff 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -40,19 +40,15 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 DATABASE_PASSWORD=please_set_this_to_some_secure_value
 
 # SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will
-# fail to communicate and can silently drop traffic. Set USE_LETSENCRYPT="yes"
-# to use the Let's Encrypt salt formula
-# (https://github.com/saltstack-formulas/letsencrypt-formula) to automatically
-# obtain and install SSL certificates for your hostname(s).
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
 #
-# Alternatively, set this variable to "no" and provide and upload your own
-# certificates to the instances and modify the 'nginx_*' salt pillars
-# accordingly
-USE_LETSENCRYPT="no"
+# See https://doc.arvados.org/intall/salt-single-host.html#certificates for more information.
+SSL_MODE="self-signed"
 
 # If you going to provide your own certificates for Arvados, the provision script can
-# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
+# help you deploy them. In order to do that, you need to set `SSL_MODE=bring-your-own` above,
 # and copy the required certificates under the directory specified in the next line.
 # The certs will be copied from this directory by the provision script.
 # Please set it to the FULL PATH to the certs dir if you're going to use a different dir
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index d09cdb2ef0..de5af681ee 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -49,16 +49,12 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 DATABASE_PASSWORD=please_set_this_to_some_secure_value
 
 # SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will
-# fail to communicate and can silently drop traffic. Set USE_LETSENCRYPT="yes"
-# to use the Let's Encrypt salt formula
-# (https://github.com/saltstack-formulas/letsencrypt-formula) to automatically
-# obtain and install SSL certificates for your hostname(s).
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
 #
-# Alternatively, set this variable to "no" and provide and upload your own
-# certificates to the instances and modify the 'nginx_*' salt pillars
-# accordingly
-USE_LETSENCRYPT="no"
+# See https://doc.arvados.org/intall/salt-single-host.html#certificates for more information.
+SSL_MODE="self-signed"
 
 # The directory to check for the config files (pillars, states) you want to use.
 # There are a few examples under 'config_examples'.
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index b323a5a835..91865ef5d5 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -185,7 +185,7 @@ WEBSOCKET_EXT_SSL_PORT=8002
 WORKBENCH1_EXT_SSL_PORT=443
 WORKBENCH2_EXT_SSL_PORT=3001
 
-USE_LETSENCRYPT="no"
+SSL_MODE="self-signed"
 CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
 
 ## These are ARVADOS-related parameters
@@ -478,18 +478,19 @@ EOFPSLS
 
 # States, extra states
 if [ -d "${F_DIR}"/extra/extra ]; then
-  if [ "$DEV_MODE" = "yes" ]; then
+  SKIP_SNAKE_OIL="snakeoil_certs"
+
+  if [[ "$DEV_MODE" = "yes" || "${SSL_MODE}" == "self-signed" ]] ; then
     # In dev mode, we create some snake oil certs that we'll
-    # use as CUSTOM_CERTS, so we don't skip the states file
+    # use as CUSTOM_CERTS, so we don't skip the states file.
+    # Same when using self-signed certificates.
     SKIP_SNAKE_OIL="dont_snakeoil_certs"
-  else
-    SKIP_SNAKE_OIL="snakeoil_certs"
   fi
   for f in $(ls "${F_DIR}"/extra/extra/*.sls | grep -v ${SKIP_SNAKE_OIL}); do
   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
   done
-  # Use custom certs
-  if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+  # Use byo or self-signed certificates
+  if [ "${SSL_MODE}" != "lets-encrypt" ]; then
     mkdir -p "${F_DIR}"/extra/extra/files
   fi
 fi
@@ -500,8 +501,8 @@ if [ -z "${ROLES}" ]; then
   # States
   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
   # Currently, only available on config_examples/multi_host/aws
-  if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-    if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+  if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+    if [ "${USE_LETSENCRYPT_IAM_USER}" != "yes" ]; then
       grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - extra.aws_credentials" >> ${S_DIR}/top.sls
     fi
     grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
@@ -534,8 +535,8 @@ if [ -z "${ROLES}" ]; then
   echo "    - postgresql" >> ${P_DIR}/top.sls
 
   # Currently, only available on config_examples/multi_host/aws
-  if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-    if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+  if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+    if [ "${USE_LETSENCRYPT_IAM_USER}" != "yes" ]; then
       grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
     fi
     grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
@@ -555,6 +556,7 @@ if [ -z "${ROLES}" ]; then
     echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
 
     for c in controller websocket workbench workbench2 webshell download collections keepproxy; do
+      copy_custom_cert ${CUSTOM_CERTS_DIR} $c
       grep -q ${c} ${P_DIR}/extra_custom_certs.sls || echo "  - ${c}" >> ${P_DIR}/extra_custom_certs.sls
 
       # As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
@@ -571,7 +573,7 @@ else
   grep -q "custom_certs"    ${S_DIR}/top.sls || echo "    - extra.custom_certs" >> ${S_DIR}/top.sls
 
   # And we add the basic part for the certs pillar
-  if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+  if [ "${SSL_MODE}" != "lets-encrypt" ]; then
     # And add the certs in the custom_certs pillar
     echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
     echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
@@ -594,8 +596,8 @@ else
         ### If we don't install and run LE before arvados-api-server, it fails and breaks everything
         ### after it. So we add this here as we are, after all, sharing the host for api and controller
         # Currently, only available on config_examples/multi_host/aws
-        if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-          if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+        if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+          if [ "${USE_LETSENCRYPT_IAM_USER}" != "yes" ]; then
             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
           fi
           grep -q "letsencrypt" ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
@@ -615,7 +617,7 @@ else
         # States
         grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
         # Currently, only available on config_examples/multi_host/aws
-        if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
+        if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
           fi
@@ -643,8 +645,8 @@ else
         fi
 
         # Currently, only available on config_examples/multi_host/aws
-        if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-          if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+        if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+          if [ "${USE_LETSENCRYPT_IAM_USER}" != "yes" ]; then
             grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
           fi
           grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index a47294b3bd..5734837f3e 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -69,4 +69,4 @@ echo "Switching to user '__INITIAL_USER__'"
 export ARVADOS_API_TOKEN="${user_api_token}"
 
 echo "Running test CWL workflow"
-cwl-runner --debug hasher-workflow.cwl hasher-workflow-job.yml
+cwl-runner --local --debug hasher-workflow.cwl hasher-workflow-job.yml
-- 
2.30.2