From f43a1f883619d2ec25439145b9b35fec1265b0cc Mon Sep 17 00:00:00 2001 From: =?utf8?q?Javier=20B=C3=A9rtoli?= Date: Fri, 16 Oct 2020 13:25:31 -0300 Subject: [PATCH] feat(provision): example configurations in salt MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit refs #16471 and #16992 Arvados-DCO-1.1-Signed-off-by: Javier Bértoli --- .../install-using-salt.html.textile.liquid | 84 +++++---- tools/salt-install/Vagrantfile | 8 +- tools/salt-install/provision.sh | 123 +++++++++++-- tools/salt-install/single_host/arvados.sls | 162 ++++++++++++++++++ tools/salt-install/single_host/locale.sls | 14 ++ .../single_host/nginx_api_configuration.sls | 28 +++ .../nginx_controller_configuration.sls | 59 +++++++ .../nginx_keepproxy_configuration.sls | 58 +++++++ .../nginx_keepweb_configuration.sls | 58 +++++++ .../single_host/nginx_passenger.sls | 24 +++ .../nginx_webshell_configuration.sls | 75 ++++++++ .../nginx_websocket_configuration.sls | 59 +++++++ .../nginx_workbench2_configuration.sls | 49 ++++++ .../nginx_workbench_configuration.sls | 74 ++++++++ tools/salt-install/single_host/postgresql.sls | 42 +++++ 15 files changed, 869 insertions(+), 48 deletions(-) create mode 100644 tools/salt-install/single_host/arvados.sls create mode 100644 tools/salt-install/single_host/locale.sls create mode 100644 tools/salt-install/single_host/nginx_api_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_controller_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_keepproxy_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_keepweb_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_passenger.sls create mode 100644 tools/salt-install/single_host/nginx_webshell_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_websocket_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_workbench2_configuration.sls create mode 100644 tools/salt-install/single_host/nginx_workbench_configuration.sls create mode 100644 tools/salt-install/single_host/postgresql.sls diff --git a/doc/install/install-using-salt.html.textile.liquid b/doc/install/install-using-salt.html.textile.liquid index d57658ab7b..629b760ae1 100644 --- a/doc/install/install-using-salt.html.textile.liquid +++ b/doc/install/install-using-salt.html.textile.liquid @@ -13,49 +13,39 @@ SPDX-License-Identifier: CC-BY-SA-3.0 # "Introduction":#introduction # "Install Saltstack":#saltstack # "Install dependencies":#dependencies -# "Install Arvados":#arvados +# "Install Arvados using Saltstack":#saltstack +# "Single host install using the provision.sh script":#single_host +# "Local testing Arvados in a Vagrant box":#vagrant h2(#tlrl). For the impatient (TL;RL) -We added a "script":https://github.com/arvados/arvados/tree/master/tools/salt-install/provision.sh -under the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory. +We added a "script":https://github.com/arvados/arvados/tree/master/tools/salt-install/provision.sh under the "tools/salt-install":https://github.com/arvados/arvados/tree/master/tools/salt-install directory. -Just edit the @provision.sh@ script, modify the variables at the top of the file to suit your needs -(usually not required), copy the file to the host where you want to install Arvados and run it as root +Just edit the @provision.sh@ script, modify the variables at the top of the file to suit your needs (usually not required), copy the file to the host where you want to install Arvados and run it as root -

-bash ./provision.sh
+
sudo bash ./provision.sh
 

 
 
 It will install all the main Arvados components to get you up and running.
 
-There is a companion @Vagrantfile@ to install Arvados in a local Vagrant virtual machine. If you have
-"Vagrant":https://www.vagrantup.com/ installed in your computer, you can just get Arvados installed
-running in a local virtual machine with the command
+There is a companion @Vagrantfile@ to install Arvados in a local Vagrant virtual machine. If you have "Vagrant":https://www.vagrantup.com/ installed in your computer, you can just get Arvados installed running in a local virtual machine with the command
 
 
-

-vagrant up
+
vagrant up
 

 
 
-If you want to run these scripts in another host or instance, just copy the @provision.sh@ file to the
-desired host and run it.
+If you want to run these scripts in another host or instance, just copy the directory to the destination host, edit the @provision.sh@ file and run it.
 
 The rest of the document is a description of the individual steps the @provision.sh@ script perform.
 
 h2(#introduction). Introduction
 
-To ease the installation of the various Arvados components, we have developed a
-"Saltstack":https://www.saltstack.com/ 's
-"arvados-formula":https://github.com/saltstack-formulas/arvados-formula which
-can help you get an Arvados cluster up and running.
+To ease the installation of the various Arvados components, we have developed a "Saltstack":https://www.saltstack.com/ 's "arvados-formula":https://github.com/saltstack-formulas/arvados-formula which can help you get an Arvados cluster up and running.
 
-Saltstack is a Python-based, open-source software for event-driven IT
-automation, remote task execution, and configuration management. It can be used
-in a master/minion setup or master-less.
+Saltstack is a Python-based, open-source software for event-driven IT automation, remote task execution, and configuration management. It can be used in a master/minion setup or master-less.
 
 h2(#saltstack). Install Saltsack
 
@@ -64,8 +54,7 @@ If you already have a Saltstack environment you can skip this section.
 The simplest way to get Salt up and running on a node is to use the bootstrap script they provide:
 
 
-

-curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
+
curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
 sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
 

 
@@ -74,8 +63,7 @@ For more information check "Saltstack's documentation":https://docs.saltstack.co
 
 h2(#dependencies). Install dependencies
 
-Arvados depends in a few applications and packages (postgresql, nginx+passenger, ruby) that can also be
-installed using their respective Saltstack formulas.
+Arvados depends in a few applications and packages (postgresql, nginx+passenger, ruby) that can also be installed using their respective Saltstack formulas.
 
 The formulas we use are:
 
@@ -84,15 +72,11 @@ The formulas we use are:
 * "docker":https://github.com/saltstack-formulas/docker-formula.git
 * "locale":https://github.com/saltstack-formulas/locale-formula.git
 
-There are example Salt pillar files for each of those formulas in the
-test/salt/pillar/examples":https://github.com/saltstack-formulas/arvados-formula/tree/master/test/salt/pillar/examples
-directory. As they are, they allow you to get all the main Arvados components up and running
-in a single node.
+There are example Salt pillar files for each of those formulas in the test/salt/pillar/examples":https://github.com/saltstack-formulas/arvados-formula/tree/master/test/salt/pillar/examples directory. As they are, they allow you to get all the main Arvados components up and running.
 
-h2(#arvados). Install Arvados
+h2(#saltstack). Install Arvados using Saltstack
 
-The Arvados formula we maintain is located in the Saltstack's community repository of
-formulas:
+The Arvados formula we maintain is located in the Saltstack's community repository of formulas:
 
 * "arvados-formula":https://github.com/saltstack-formulas/arvados-formula.git
 
@@ -101,3 +85,39 @@ and we keep a @development@ version at our own repo
 * "arvados-formula develpment":https://github.com/arvados/arvados-formula.git
 
 This last one might break from time to time, as we try and add new features. Use with caution.
+
+As much as possible, we try to keep it up to date, with example pillars to help you deploy Arvados.
+
+For those familiar with Saltstack, the process to get it deployed is similar to any other formula:
+
+1. Fork/copy the formula to your Salt master host.
+2. Edit the Arvados, nginx, postgres, locale and docker pillars to match your desired configuration.
+3. Run a @state.apply@ to get it deployed.
+
+h2(#single_host). Single host install using the provision.sh script
+
+In case you want to run it on a single node and don't want to (or don't have) a Saltstack master/minion infrastructure in place, we provide a very simple script named @provision.sh@ which can be used to deploy Arvados, using the @arvados-formula@ in a master-less setup.
+
+Just
+
+1. edit the variables at the very beginning ot the file,
+2. run the script
+3. wait for it to finish
+
+If everything goes OK, you'll get some final lines stating something like:
+
+
+
arvados: Succeeded: 109 (changed=9)
+arvados: Failed:      0
+

+
+
+h2(#vagrant). Local testing Arvados in a Vagrant box
+
+If you want to try Arvados in a local virtual machine, there's a companion @Vagrantfile@ to install it on a virtual machine in your computer using "Vagrant":https://www.vagrantup.com/.
+
+To get it running, simply install Vagrant in your computer, edit the @provision@ file as described in the previous section, and run
+
+
vagrant up
+

+
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index 36f6827617..93bb77d4fa 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -13,7 +13,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   config.vm.define "arvados" do |arv|
     arv.vm.box = "bento/debian-10"
-    arv.vm.hostname = "arv.local"
+    arv.vm.hostname = "arva2.arv.local"
     # Networking
     arv.vm.network "forwarded_port", guest: 8443, host: 8443
     arv.vm.network "forwarded_port", guest: 25100, host: 25100
@@ -28,6 +28,10 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     # arv.vm.synced_folder "salt_pillars", "/srv/pillars",
     #                      create: true
     arv.vm.provision "shell",
-                     path: "provision.sh"
+                     path: "provision.sh",
+                     args: [
+                       "--vagrant",
+                       "--ssl-port=8443"
+                     ].join(" ")
   end
 end
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 228b506645..1ee13752b0 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/bin/bash -x
 
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
@@ -15,11 +15,21 @@
 CLUSTER="arva2"
 DOMAIN="arv.local"
 
+# The example config you want to use. Currently, only "single_host" is
+# available
+CONFIG_DIR="single_host"
+
 # Which release of Arvados repo you want to use
 RELEASE="production"
 # Which version of Arvados you want to install. Defaults to 'latest'
 # in the desired repo
-# VERSION="2.0.4"
+VERSION="latest"
+
+# Host SSL port where you want to point your browser to access Arvados
+# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
+# You can point it to another port if desired
+# In Vagrant, make sure it matches what you set in the Vagrantfile
+# HOST_SSL_PORT=443
 
 # This is a arvados-formula setting. 
 # If branch is set, the script will switch to it before running salt
@@ -29,6 +39,55 @@ RELEASE="production"
 ##########################################################
 # Usually there's no need to modify things below this line
 
+set -o pipefail
+
+usage() {
+  echo >&2
+  echo >&2 "Usage: $0 [-h] [-h]"
+  echo >&2
+  echo >&2 "$0 options:"
+  echo >&2 "  -v, --vagrant           Run in vagrant and use the /vagrant shared dir"
+  echo >&2 "  -p , --ssl-port   SSL port to use for the web applications"
+  echo >&2 "  -h, --help              Display this help and exit"
+  echo >&2
+}
+
+arguments() {
+  # NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
+  TEMP=`getopt -o hvp: \
+    --long help,vagrant,ssl-port: \
+    -n "$0" -- "$@"`
+
+  if [ $? != 0 ] ; then echo "GNU getopt missing? Use -h for help"; exit 1 ; fi
+  # Note the quotes around `$TEMP': they are essential!
+  eval set -- "$TEMP"
+
+  while [ $# -ge 1 ]; do
+    case $1 in
+      -v | --vagrant)
+        VAGRANT="yes"
+        shift
+        ;;
+      -p | --ssl-port)
+        HOST_SSL_PORT=${2}
+        shift 2
+        ;;
+      --)
+        shift
+        break
+        ;;
+      *)
+        usage
+        exit 1
+        ;;
+    esac
+  done
+}
+
+HOST_SSL_PORT=443
+
+arguments $@
+
 # Salt's dir
 ## states
 S_DIR="/srv/salt"
@@ -36,19 +95,17 @@ S_DIR="/srv/salt"
 F_DIR="/srv/formulas"
 ##pillars
 P_DIR="/srv/pillars"
-# In vagrant, we can use the shared dir
-# P_DIR="/vagrant/salt_pillars"
 
-sudo apt-get update
-sudo apt-get install -y curl git
+apt-get update
+apt-get install -y curl git
 
 dpkg -l |grep salt-minion
 if [ ${?} -eq 0 ]; then
   echo "Salt already installed"
 else
   curl -L https://bootstrap.saltstack.com -o /tmp/bootstrap_salt.sh
-  sudo sh /tmp/bootstrap_salt.sh -XUdfP -x python3
-  sudo /bin/systemctl disable salt-minion.service
+  sh /tmp/bootstrap_salt.sh -XUdfP -x python3
+  /bin/systemctl disable salt-minion.service
 fi
 
 # Set salt to masterless mode
@@ -103,7 +160,8 @@ EOFPSLS
 # Get the formula and dependencies
 cd ${F_DIR} || exit 1
 for f in postgres arvados nginx docker locale; do
-  git clone https://github.com/saltstack-formulas/${f}-formula.git
+  # git clone https://github.com/saltstack-formulas/${f}-formula.git
+  git clone https://github.com/netmanagers/${f}-formula.git
 done
 
 if [ "x${BRANCH}" != "x" ]; then
@@ -112,18 +170,55 @@ if [ "x${BRANCH}" != "x" ]; then
   cd -
 fi
 
-sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g; s/release: development/release: ${RELEASE}/g; s/# version: '2.0.4'/version: '${VERSION}'/g" \
-  ${F_DIR}/arvados-formula/test/salt/pillar/arvados_dev.sls > ${P_DIR}/arvados.sls
+# sed "s/__DOMAIN__/${DOMAIN}/g; s/__CLUSTER__/${CLUSTER}/g; s/__RELEASE__/${RELEASE}/g; s/__VERSION__/${VERSION}/g" \
+#   ${CONFIG_DIR}/arvados_dev.sls > ${P_DIR}/arvados.sls
+
+if [ "x${VAGRANT}" = "xyes" ]; then
+  SOURCE_PILLARS_DIR="/vagrant/${CONFIG_DIR}"
+else
+  SOURCE_PILLARS_DIR="./${CONFIG_DIR}"
+fi
 
 # Replace cluster and domain name in the example pillars
-for f in ${F_DIR}/arvados-formula/test/salt/pillar/examples/*; do
-  sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
+for f in ${SOURCE_PILLARS_DIR}/*; do
+  # sed "s/example.net/${DOMAIN}/g; s/fixme/${CLUSTER}/g" \
+  sed "s/__DOMAIN__/${DOMAIN}/g;
+       s/__CLUSTER__/${CLUSTER}/g;
+       s/__RELEASE__/${RELEASE}/g;
+       s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
+       s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
+       s/__VERSION__/${VERSION}/g" \
   ${f} > ${P_DIR}/$(basename ${f})
 done
 
-# Let's write a /etc/hosts file that points all the hosts to localhost
+# Let's write an /etc/hosts file that points all the hosts to localhost
 
 echo "127.0.0.2 api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
 
+# FIXME! #16992 Temporary fix for psql call in arvados-api-server
+if [ -e /root/.psqlrc ]; then
+  if ! ( grep 'pset pager off' /root/.psqlrc ); then
+    RESTORE_PSQL="yes"
+    cp /root/.psqlrc /root/.psqlrc.provision.backup
+  fi
+else
+  DELETE_PSQL="yes"
+fi
+
+echo '\pset pager off' >> /root/.psqlrc
+# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
+
 # Now run the install
 salt-call --local state.apply -l debug
+
+# FIXME! #16992 Temporary fix for psql call in arvados-api-server
+if [ "x${DELETE_PSQL}" = "xyes" ]; then
+  echo "Removing .psql file"
+  rm /root/.psqlrc
+fi
+
+if [ "x${RESTORE_PSQL}" = "xyes" ]; then
+  echo "Restroting .psql file"
+  mv -v /root/.psqlrc.provision.backup /root/.psqlrc
+fi
+# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
diff --git a/tools/salt-install/single_host/arvados.sls b/tools/salt-install/single_host/arvados.sls
new file mode 100644
index 0000000000..9b1e41d46e
--- /dev/null
+++ b/tools/salt-install/single_host/arvados.sls
@@ -0,0 +1,162 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The variables commented out are the default values that the formula uses.
+# The uncommented values are REQUIRED values. If you don't set them, running
+# this formula will fail.
+arvados:
+  ### GENERAL CONFIG
+  version: '__VERSION__'
+  ## It makes little sense to disable this flag, but you can, if you want :)
+  # use_upstream_repo: true
+
+  ## Repo URL is built with grains values. If desired, it can be completely
+  ## overwritten with the pillar parameter 'repo_url'
+  # repo:
+  #   humanname: Arvados Official Repository
+
+  release: __RELEASE__
+
+  ## IMPORTANT!!!!!
+  ## api, workbench and shell require some gems, so you need to make sure ruby
+  ## and deps are installed in order to install and compile the gems.
+  ## We default to `false` in these two variables as it's expected you already
+  ## manage OS packages with some other tool and you don't want us messing up
+  ## with your setup.
+  ruby:
+    ## We set these to `true` here for testing purposes.
+    ## They both default to `false`.
+    manage_ruby: true
+    manage_gems_deps: true
+    # pkg: ruby
+    # gems_deps:
+    #     - curl
+    #     - g++
+    #     - gcc
+    #     - git
+    #     - libcurl4
+    #     - libcurl4-gnutls-dev
+    #     - libpq-dev
+    #     - libxml2
+    #     - libxml2-dev
+    #     - make
+    #     - python3-dev
+    #     - ruby-dev
+    #     - zlib1g-dev
+
+  # config:
+  #   file: /etc/arvados/config.yml
+  #   user: root
+  ## IMPORTANT!!!!!
+  ## If you're intalling any of the rails apps (api, workbench), the group
+  ## should be set to that of the web server, usually `www-data`
+  #   group: root
+  #   mode: 640
+
+  ### ARVADOS CLUSTER CONFIG
+  cluster:
+    name: __CLUSTER__
+    domain: __DOMAIN__
+
+    database:
+      # max concurrent connections per arvados server daemon
+      # connection_pool_max: 32
+      name: arvados
+      host: 127.0.0.1
+      password: changeme_arvados
+      user: arvados
+      encoding: en_US.utf8
+      client_encoding: UTF8
+
+    tls:
+      # certificate: ''
+      # key: ''
+      # required to test with snakeoil certs
+      insecure: true
+
+    ### TOKENS
+    tokens:
+      system_root: changeme_system_root_token
+      management: changeme_management_token
+      rails_secret: changeme_rails_secret_token
+      anonymous_user: changeme_anonymous_user_token
+
+    ### KEYS
+    secrets:
+      blob_signing_key: changeme_blob_signing_key
+      workbench_secret_key: changeme_workbench_secret_key
+      dispatcher_access_key: changeme_dispatcher_access_key
+      dispatcher_secret_key: changeme_dispatcher_secret_key
+      keep_access_key: changeme_keep_access_key
+      keep_secret_key: changeme_keep_secret_key
+
+    Login:
+      Test:
+        Enable: true
+        javier:
+          User: javier@arva2.arv.local
+          Password: perico
+
+    AuditLogs:
+      Section_to_ignore:
+        - some_random_value
+
+    ### VOLUMES
+    ## This should usually match all your `keepstore` instances
+    Volumes:
+      # the volume name will be composed with
+      # -nyw5e-
+      __CLUSTER__-nyw5e-000000000000000:
+        AccessViaHosts:
+          http://keep0.__CLUSTER__.__DOMAIN__:25107:
+            ReadOnly: false
+        Replication: 2
+        Driver: Directory
+        DriverParameters:
+          Root: /tmp
+
+    Users:
+      NewUsersAreActive: true
+      AutoAdminFirstUser: true
+      AutoSetupNewUsers: true
+      AutoSetupNewUsersWithRepository: true
+
+    Services:
+      Controller:
+        ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        InternalURLs:
+          http://127.0.0.2:8003: {}
+      DispatchCloud:
+        InternalURLs:
+          http://__CLUSTER__.__DOMAIN__:9006: {}
+      Keepbalance:
+        InternalURLs:
+          http://__CLUSTER__.__DOMAIN__:9005: {}
+      Keepproxy:
+        ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        InternalURLs:
+          http://127.0.0.2:25100: {}
+      Keepstore:
+        InternalURLs:
+          http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
+      RailsAPI:
+        InternalURLs:
+          http://127.0.0.2:8004: {}
+      WebDAV:
+        ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        InternalURLs:
+          http://127.0.0.2:9002: {}
+      WebDAVDownload:
+        ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+      WebShell:
+        ExternalURL: https://webshell.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+      Websocket:
+        ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
+        InternalURLs:
+          http://127.0.0.2:8005: {}
+      Workbench1:
+        ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+      Workbench2:
+        ExternalURL: https://workbench2.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
diff --git a/tools/salt-install/single_host/locale.sls b/tools/salt-install/single_host/locale.sls
new file mode 100644
index 0000000000..17f53a2881
--- /dev/null
+++ b/tools/salt-install/single_host/locale.sls
@@ -0,0 +1,14 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+locale:
+  present:
+    - "en_US.UTF-8 UTF-8"
+  default:
+    # Note: On debian systems don't write the second 'UTF-8' here or you will
+    # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8
+    # Restart the minion after you corrected this!
+    name: 'en_US.UTF-8'
+    requires: 'en_US.UTF-8 UTF-8'
diff --git a/tools/salt-install/single_host/nginx_api_configuration.sls b/tools/salt-install/single_host/nginx_api_configuration.sls
new file mode 100644
index 0000000000..db0bea126e
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_api_configuration.sls
@@ -0,0 +1,28 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+  config:
+    group: www-data
+
+### NGINX
+nginx:
+  ### SITES
+  servers:
+    managed:
+      arvados_api:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - listen: '127.0.0.2:8004'
+            - server_name: api
+            - root: /var/www/arvados-api/current/public
+            - index:  index.html index.htm
+            - access_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+            - error_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.error.log
+            - passenger_enabled: 'on'
+            - client_max_body_size: 128m
diff --git a/tools/salt-install/single_host/nginx_controller_configuration.sls b/tools/salt-install/single_host/nginx_controller_configuration.sls
new file mode 100644
index 0000000000..2b2e7d5915
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_controller_configuration.sls
@@ -0,0 +1,59 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        'geo $external_client':
+          default: 1
+          '127.0.0.0/8': 0
+        upstream controller_upstream:
+          - server: '127.0.0.2:8003  fail_timeout=10s'
+
+  ### SITES
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_controller_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __CLUSTER__.__DOMAIN__
+            - listen:
+              - 80 default
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_controller_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://controller_upstream'
+              - proxy_read_timeout: 300
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_set_header: 'X-External-Client $external_client'
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
+            - client_max_body_size: 128m
diff --git a/tools/salt-install/single_host/nginx_keepproxy_configuration.sls b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
new file mode 100644
index 0000000000..29cd0cb440
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_keepproxy_configuration.sls
@@ -0,0 +1,58 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        upstream keepproxy_upstream:
+          - server: '127.0.0.2:25100 fail_timeout=10s'
+
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_keepproxy_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: keep.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_keepproxy_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: keep.__CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://keepproxy_upstream'
+              - proxy_read_timeout: 90
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_buffering: 'off'
+            - client_body_buffer_size: 64M
+            - client_max_body_size: 64M
+            - proxy_http_version: '1.1'
+            - proxy_request_buffering: 'off'
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_keepweb_configuration.sls b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
new file mode 100644
index 0000000000..bd0a636b0e
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_keepweb_configuration.sls
@@ -0,0 +1,58 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        upstream collections_downloads_upstream:
+          - server: '127.0.0.2:9002 fail_timeout=10s'
+
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_collections_download_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      ### COLLECTIONS / DOWNLOAD
+      arvados_collections_download_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: collections.__CLUSTER__.__DOMAIN__ download.__CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://collections_downloads_upstream'
+              - proxy_read_timeout: 90
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_buffering: 'off'
+            - client_max_body_size: 0
+            - proxy_http_version: '1.1'
+            - proxy_request_buffering: 'off'
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_passenger.sls b/tools/salt-install/single_host/nginx_passenger.sls
new file mode 100644
index 0000000000..6ce75faa70
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_passenger.sls
@@ -0,0 +1,24 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  install_from_phusionpassenger: true
+  lookup:
+    passenger_package: libnginx-mod-http-passenger
+    passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+
+  ### SERVER
+  server:
+    config:
+      include: 'modules-enabled/*.conf'
+      worker_processes: 4
+
+  ### SITES
+  servers:
+    managed:
+      # Remove default webserver
+      default:
+        enabled: false
diff --git a/tools/salt-install/single_host/nginx_webshell_configuration.sls b/tools/salt-install/single_host/nginx_webshell_configuration.sls
new file mode 100644
index 0000000000..e33ddcea70
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_webshell_configuration.sls
@@ -0,0 +1,75 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+
+      ### STREAMS
+      http:
+        upstream webshell_upstream:
+          - server: '127.0.0.2:4200 fail_timeout=10s'
+
+  ### SITES
+  servers:
+    managed:
+      arvados_webshell_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: webshell.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_webshell_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: webshell.__CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /shell.__CLUSTER__.__DOMAIN__:
+              - proxy_pass: 'http://webshell_upstream'
+              - proxy_read_timeout: 90
+              - proxy_connect_timeout: 90
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_ssl_session_reuse: 'off'
+
+              - "if ($request_method = 'OPTIONS')":
+                - add_header: "'Access-Control-Allow-Origin' '*'"
+                - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+                - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+                - add_header: "'Access-Control-Max-Age' 1728000"
+                - add_header: "'Content-Type' 'text/plain charset=UTF-8'"
+                - add_header: "'Content-Length' 0"
+                - return: 204
+
+              - "if ($request_method = 'POST')":
+                - add_header: "'Access-Control-Allow-Origin' '*'"
+                - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+                - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+              - "if ($request_method = 'GET')":
+                - add_header: "'Access-Control-Allow-Origin' '*'"
+                - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+                - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
+
diff --git a/tools/salt-install/single_host/nginx_websocket_configuration.sls b/tools/salt-install/single_host/nginx_websocket_configuration.sls
new file mode 100644
index 0000000000..2241d3b8ea
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_websocket_configuration.sls
@@ -0,0 +1,59 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        upstream websocket_upstream:
+          - server: '127.0.0.2:8005 fail_timeout=10s'
+
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_websocket_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: ws.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_websocket_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: ws.__CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://websocket_upstream'
+              - proxy_read_timeout: 600
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: 'Host $host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'Upgrade $http_upgrade'
+              - proxy_set_header: 'Connection "upgrade"'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_buffering: 'off'
+            - client_body_buffer_size: 64M
+            - client_max_body_size: 64M
+            - proxy_http_version: '1.1'
+            - proxy_request_buffering: 'off'
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench2_configuration.sls b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
new file mode 100644
index 0000000000..733397adf3
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_workbench2_configuration.sls
@@ -0,0 +1,49 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+  config:
+    group: www-data
+
+### NGINX
+nginx:
+  ### SITES
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_workbench2_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench2.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_workbench2_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench2.__CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - root: /var/www/arvados-workbench2/workbench2
+              - try_files: '$uri $uri/ /index.html'
+              - 'if (-f $document_root/maintenance.html)':
+                - return: 503
+            - location /config.json:
+              - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__"}' ~ "'" }}
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/single_host/nginx_workbench_configuration.sls b/tools/salt-install/single_host/nginx_workbench_configuration.sls
new file mode 100644
index 0000000000..76fb134385
--- /dev/null
+++ b/tools/salt-install/single_host/nginx_workbench_configuration.sls
@@ -0,0 +1,74 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+  config:
+    group: www-data
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+
+      ### STREAMS
+      http:
+        upstream workbench_upstream:
+          - server: '127.0.0.2:9000 fail_timeout=10s'
+
+  ### SITES
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_workbench_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_workbench_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench.__CLUSTER__.__DOMAIN__
+            - listen:
+              - __HOST_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://workbench_upstream'
+              - proxy_read_timeout: 300
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+            # - include: 'snippets/letsencrypt.conf'
+            - include: 'snippets/snakeoil.conf'
+            - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
+
+      arvados_workbench_upstream:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - listen: '127.0.0.2:9000'
+            - server_name: workbench
+            - root: /var/www/arvados-workbench/current/public
+            - index:  index.html index.htm
+            - passenger_enabled: 'on'
+            # yamllint disable-line rule:line-length
+            - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+            - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.error.log
diff --git a/tools/salt-install/single_host/postgresql.sls b/tools/salt-install/single_host/postgresql.sls
new file mode 100644
index 0000000000..56b0a42e8b
--- /dev/null
+++ b/tools/salt-install/single_host/postgresql.sls
@@ -0,0 +1,42 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### POSTGRESQL
+postgres:
+  use_upstream_repo: false
+  pkgs_extra:
+    - postgresql-contrib
+  postgresconf: |-
+    listen_addresses = '*'  # listen on all interfaces
+  acls:
+    - ['local', 'all', 'postgres', 'peer']
+    - ['local', 'all', 'all', 'peer']
+    - ['host', 'all', 'all', '127.0.0.1/32', 'md5']
+    - ['host', 'all', 'all', '::1/128', 'md5']
+    - ['host', 'arvados', 'arvados', '127.0.0.1/32']
+  users:
+    arvados:
+      ensure: present
+      password: changeme_arvados
+
+  # tablespaces:
+  #   arvados_tablespace:
+  #     directory: /path/to/some/tbspace/arvados_tbsp
+  #     owner: arvados
+
+  databases:
+    arvados:
+      owner: arvados
+      template: template0
+      lc_ctype: en_US.utf8
+      lc_collate: en_US.utf8
+      # tablespace: arvados_tablespace
+      schemas:
+        public:
+          owner: arvados
+      extensions:
+        pg_trgm:
+          if_not_exists: true
+          schema: public
-- 
2.30.2