From 9191a9a512d1044ea1efc5d5477412097d367a4e Mon Sep 17 00:00:00 2001 From: =?utf8?q?Javier=20B=C3=A9rtoli?= Date: Mon, 22 Mar 2021 13:32:22 -0300 Subject: [PATCH] fix(provision): add multi hosts installation examples MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit refs #17246 Arvados-DCO-1.1-Signed-off-by: Javier Bértoli --- .../salt-multi-host.html.textile.liquid | 3 +- .../config_examples/multi_host/aws/README.md | 9 + .../multi_host/aws/certs/README.md | 19 ++ .../multi_host/aws/pillars/arvados.sls | 264 ++++++++++++++++++ .../multi_host/aws/pillars/docker.sls | 9 + .../multi_host/aws/pillars/letsencrypt.sls | 30 ++ .../letsencrypt_controller_configuration.sls | 18 ++ .../letsencrypt_keepproxy_configuration.sls | 18 ++ .../letsencrypt_keepweb_configuration.sls | 23 ++ .../letsencrypt_webshell_configuration.sls | 18 ++ .../letsencrypt_websocket_configuration.sls | 18 ++ .../letsencrypt_workbench2_configuration.sls | 18 ++ .../letsencrypt_workbench_configuration.sls | 18 ++ .../multi_host/aws/pillars/locale.sls | 14 + .../aws/pillars/nginx_api_configuration.sls | 28 ++ .../nginx_controller_configuration.sls | 61 ++++ .../pillars/nginx_keepproxy_configuration.sls | 59 ++++ .../pillars/nginx_keepweb_configuration.sls | 89 ++++++ .../aws/pillars/nginx_passenger.sls | 53 ++++ .../pillars/nginx_webshell_configuration.sls | 76 +++++ .../pillars/nginx_websocket_configuration.sls | 60 ++++ .../nginx_workbench2_configuration.sls | 50 ++++ .../pillars/nginx_workbench_configuration.sls | 75 +++++ .../multi_host/aws/pillars/postgresql.sls | 42 +++ .../multi_host/aws/states/host_entries.sls | 71 +++++ tools/terraform/.gitignore | 7 + 26 files changed, 1148 insertions(+), 2 deletions(-) create mode 100644 tools/salt-install/config_examples/multi_host/aws/README.md create mode 100644 tools/salt-install/config_examples/multi_host/aws/certs/README.md create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/docker.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_controller_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepproxy_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepweb_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_webshell_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_websocket_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench2_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/locale.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_api_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepproxy_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepweb_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_webshell_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_websocket_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench2_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench_configuration.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls create mode 100644 tools/salt-install/config_examples/multi_host/aws/states/host_entries.sls create mode 100644 tools/terraform/.gitignore diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid index 50de6e439b..709c32e2a5 100644 --- a/doc/install/salt-multi-host.html.textile.liquid +++ b/doc/install/salt-multi-host.html.textile.liquid @@ -39,6 +39,7 @@ The formulas we use are: * "nginx":https://github.com/saltstack-formulas/nginx-formula.git * "docker":https://github.com/saltstack-formulas/docker-formula.git * "locale":https://github.com/saltstack-formulas/locale-formula.git +* "letsencrypt":https://github.com/saltstack-formulas/letsencrypt-formula.git There are example Salt pillar files for each of those formulas in the "arvados-formula's test/salt/pillar/examples":https://github.com/arvados/arvados-formula/tree/master/test/salt/pillar/examples directory. As they are, they allow you to get all the main Arvados components up and running. @@ -56,8 +57,6 @@ As the Saltstack's community keeps a "repository of formulas":https://github.com there, and do our best effort to keep it in sync with ours. -A @development@ branch exists which uses Arvados' development repositories. This last one might break from time to time, as we try and add new features. As much as possible, we try to keep it up to date, with example pillars to help you deploy Arvados. Use with caution. - For those familiar with Saltstack, the process to get Arvados deployed is similar to any other formula: 1. Fork/copy the formula to your Salt master host. diff --git a/tools/salt-install/config_examples/multi_host/aws/README.md b/tools/salt-install/config_examples/multi_host/aws/README.md new file mode 100644 index 0000000000..58911d956c --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/README.md @@ -0,0 +1,9 @@ +Arvados installation using multiple instances +============================================= + +These files let you setup Arvados on multiple instances on AWS. This setup +considers deploying the instances on an isolated VPC, created/managed with +[the Arvados terraform code](https://github.com/arvados/arvados/tree/terraform/tools/terraform) +in our repo. + +Please check [the Arvados installation documentation](https://doc.arvados.org/install/salt-multi-host.html) for more details. diff --git a/tools/salt-install/config_examples/multi_host/aws/certs/README.md b/tools/salt-install/config_examples/multi_host/aws/certs/README.md new file mode 100644 index 0000000000..00d486e1cd --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/certs/README.md @@ -0,0 +1,19 @@ +SSL Certificates +================ + +Add the certificates for your hosts in this directory. + +The nodes requiring certificates are: + +* CLUSTER.DOMAIN +* collections.CLUSTER.DOMAIN +* \*\-\-collections.CLUSTER.DOMAIN +* download.CLUSTER.DOMAIN +* keep.CLUSTER.DOMAIN +* workbench.CLUSTER.DOMAIN +* workbench2.CLUSTER.DOMAIN +* ws.CLUSTER.DOMAIN + +They can be individual certificates or a wildcard certificate for all of them. + +Please remember to modify the *nginx\_\** salt pillars accordingly. diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls new file mode 100644 index 0000000000..4ecc65e28f --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls @@ -0,0 +1,264 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +# The variables commented out are the default values that the formula uses. +# The uncommented values are REQUIRED values. If you don't set them, running +# this formula will fail. +arvados: + ### GENERAL CONFIG + version: '__VERSION__' + ## It makes little sense to disable this flag, but you can, if you want :) + # use_upstream_repo: true + + ## Repo URL is built with grains values. If desired, it can be completely + ## overwritten with the pillar parameter 'repo_url' + # repo: + # humanname: Arvados Official Repository + + release: __RELEASE__ + + ## IMPORTANT!!!!! + ## api, workbench and shell require some gems, so you need to make sure ruby + ## and deps are installed in order to install and compile the gems. + ## We default to `false` in these two variables as it's expected you already + ## manage OS packages with some other tool and you don't want us messing up + ## with your setup. + ruby: + ## We set these to `true` here for testing purposes. + ## They both default to `false`. + manage_ruby: true + manage_gems_deps: true + # pkg: ruby + # gems_deps: + # - curl + # - g++ + # - gcc + # - git + # - libcurl4 + # - libcurl4-gnutls-dev + # - libpq-dev + # - libxml2 + # - libxml2-dev + # - make + # - python3-dev + # - ruby-dev + # - zlib1g-dev + + # config: + # file: /etc/arvados/config.yml + # user: root + ## IMPORTANT!!!!! + ## If you're intalling any of the rails apps (api, workbench), the group + ## should be set to that of the web server, usually `www-data` + # group: root + # mode: 640 + dispatcher: + pkg: + name: arvados-dispatch-cloud + service: + name: arvados-dispatch-cloud + + ### ARVADOS CLUSTER CONFIG + cluster: + name: __CLUSTER__ + domain: __DOMAIN__ + + database: + # max concurrent connections per arvados server daemon + # connection_pool_max: 32 + name: __CLUSTER___arvados + host: __DATABASE_INT_IP__ + password: "__DATABASE_PASSWORD__" + user: __CLUSTER___arvados + encoding: en_US.utf8 + client_encoding: UTF8 + + tls: + # certificate: '' + # key: '' + # required to test with arvados-snakeoil certs + insecure: false + + ### TOKENS + tokens: + system_root: __SYSTEM_ROOT_TOKEN__ + management: __MANAGEMENT_TOKEN__ + anonymous_user: __ANONYMOUS_USER_TOKEN__ + + ### KEYS + secrets: + blob_signing_key: __BLOB_SIGNING_KEY__ + workbench_secret_key: __WORKBENCH_SECRET_KEY__ + + Login: + Test: + Enable: true + Users: + __INITIAL_USER__: + Email: __INITIAL_USER_EMAIL__ + Password: __INITIAL_USER_PASSWORD__ + + ### CONTAINERS + Containers: + MaxRetryAttempts: 10 + CloudVMs: + ResourceTags: + Name: __CLUSTER__-compute-node + BootProbeCommand: 'sudo docker ps -q' + ImageID: ami-FIXMEFIXMEFIXMEFI + Driver: ec2 + DriverParameters: + Region: FIXME + EBSVolumeType: gp2 + AdminUsername: FIXME + ### This SG should allow SSH from the dispatcher to the compute nodes + SecurityGroupIDs: ['sg-FIXMEFIXMEFIXMEFI'] + SubnetID: subnet-FIXMEFIXMEFIXMEFI + DispatchPrivateKey: | + -----BEGIN OPENSSH PRIVATE KEY----- + Read https://doc.arvados.org/v2.0/install/install-dispatch-cloud.html + for details on how to create it and where to place the key + FIXMEFIXMEFIXMEFI + -----END OPENSSH PRIVATE KEY----- + + ### VOLUMES + ## This should usually match all your `keepstore` instances + Volumes: + # the volume name will be composed with + # -nyw5e- + __CLUSTER__-nyw5e-0000000000000000: + AccessViaHosts: + 'http://__KEEPSTORE0_INT_IP__:25107': + ReadOnly: false + Replication: 2 + Driver: S3 + DriverParameters: + Bucket: __CLUSTER__-nyw5e-0000000000000000-volume + IAMRole: __CLUSTER__-keepstore-00-iam-role + Region: FIXME + __CLUSTER__-nyw5e-0000000000000001: + AccessViaHosts: + 'http://__KEEPSTORE1_INT_IP__:25107': + ReadOnly: false + Replication: 2 + Driver: S3 + DriverParameters: + Bucket: __CLUSTER__-nyw5e-0000000000000001-volume + IAMRole: __CLUSTER__-keepstore-01-iam-role + Region: FIXME + + Users: + NewUsersAreActive: true + AutoAdminFirstUser: true + AutoSetupNewUsers: true + AutoSetupNewUsersWithRepository: true + + Services: + Controller: + ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__' + InternalURLs: + 'http://localhost:8003': {} + DispatchCloud: + InternalURLs: + 'http://__CONTROLLER_INT_IP__:9006': {} + Keepproxy: + ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__KEEP_EXT_SSL_PORT__' + InternalURLs: + 'http://localhost:25107': {} + Keepstore: + InternalURLs: + 'http://__KEEPSTORE0_INT_IP__:25107': {} + 'http://__KEEPSTORE1_INT_IP__:25107': {} + RailsAPI: + InternalURLs: + 'http://localhost:8004': {} + WebDAV: + ExternalURL: 'https://*--collections.__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__/' + InternalURLs: + 'http://localhost:9002': {} + WebDAVDownload: + ExternalURL: 'https://download.__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__' + WebShell: + ExternalURL: 'https://webshell.__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__' + Websocket: + ExternalURL: 'wss://ws.__CLUSTER__.__DOMAIN__/websocket' + InternalURLs: + 'http://localhost:8005': {} + Workbench1: + ExternalURL: 'https://workbench.__CLUSTER__.__DOMAIN__:__WORKBENCH1_EXT_SSL_PORT__' + Workbench2: + ExternalURL: 'https://workbench2.__CLUSTER__.__DOMAIN__:__WORKBENCH2_EXT_SSL_PORT__' + + InstanceTypes: + t3small: + ProviderType: t3.small + VCPUs: 2 + RAM: 2GiB + IncludedScratch: 50GB + AddedScratch: 50GB + Price: 0.0208 + c5large: + ProviderType: c5.large + VCPUs: 2 + RAM: 4GiB + IncludedScratch: 50GB + AddedScratch: 50GB + Price: 0.085 + m5large: + ProviderType: m5.large + VCPUs: 2 + RAM: 8GiB + IncludedScratch: 50GB + AddedScratch: 50GB + Price: 0.096 + c5xlarge: + ProviderType: c5.xlarge + VCPUs: 4 + RAM: 8GiB + IncludedScratch: 100GB + AddedScratch: 100GB + Price: 0.17 + m5xlarge: + ProviderType: m5.xlarge + VCPUs: 4 + RAM: 16GiB + IncludedScratch: 100GB + AddedScratch: 100GB + Price: 0.192 + m5xlarge_extradisk: + ProviderType: m5.xlarge + VCPUs: 4 + RAM: 16GiB + IncludedScratch: 400GB + AddedScratch: 400GB + Price: 0.193 + c52xlarge: + ProviderType: c5.2xlarge + VCPUs: 8 + RAM: 16GiB + IncludedScratch: 200GB + AddedScratch: 200GB + Price: 0.34 + m52xlarge: + ProviderType: m5.2xlarge + VCPUs: 8 + RAM: 32GiB + IncludedScratch: 200GB + AddedScratch: 200GB + Price: 0.384 + c54xlarge: + ProviderType: c5.4xlarge + VCPUs: 16 + RAM: 32GiB + IncludedScratch: 400GB + AddedScratch: 400GB + Price: 0.68 + m54xlarge: + ProviderType: m5.4xlarge + VCPUs: 16 + RAM: 64GiB + IncludedScratch: 400GB + AddedScratch: 400GB + Price: 0.768 diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/docker.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/docker.sls new file mode 100644 index 0000000000..54d2256159 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/docker.sls @@ -0,0 +1,9 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +docker: + pkg: + docker: + use_upstream: package diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt.sls new file mode 100644 index 0000000000..8906ac073f --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt.sls @@ -0,0 +1,30 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + use_package: true + pkgs: + - certbot: latest + - python3-certbot-nginx + config: + server: https://acme-staging-v02.api.letsencrypt.org/directory + email: __INITIAL_USER_EMAIL__ + authenticator: nginx + webroot-path: /var/www + agree-tos: true + keep-until-expiring: true + expand: true + max-log-backups: 0 + deploy-hook: systemctl reload nginx + +### NGINX +nginx: + ### SNIPPETS + snippets: + ### LETSENCRYPT DEFAULT PATH + letsencrypt_well_known.conf: + - location /.well-known: + - root: /var/www diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_controller_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_controller_configuration.sls new file mode 100644 index 0000000000..68c8512e76 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_controller_configuration.sls @@ -0,0 +1,18 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + __CLUSTER__.__DOMAIN__: + - __CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + __CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepproxy_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepproxy_configuration.sls new file mode 100644 index 0000000000..3056b89d4d --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepproxy_configuration.sls @@ -0,0 +1,18 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + keep.__CLUSTER__.__DOMAIN__: + - keep.__CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + keep.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/keep.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/keep.__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepweb_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepweb_configuration.sls new file mode 100644 index 0000000000..dc34ea6fd5 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_keepweb_configuration.sls @@ -0,0 +1,23 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + download.__CLUSTER__.__DOMAIN__: + - download.__CLUSTER__.__DOMAIN__ + collections.__CLUSTER__.__DOMAIN__: + - collections.__CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + download.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/download.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/download.__CLUSTER__.__DOMAIN__/privkey.pem + collections.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/collections.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/collections.__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_webshell_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_webshell_configuration.sls new file mode 100644 index 0000000000..e9d2bb018c --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_webshell_configuration.sls @@ -0,0 +1,18 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + webshell.__CLUSTER__.__DOMAIN__: + - webshell.__CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + webshell.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/webshell.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/webshell.__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_websocket_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_websocket_configuration.sls new file mode 100644 index 0000000000..d24431fac8 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_websocket_configuration.sls @@ -0,0 +1,18 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + ws.__CLUSTER__.__DOMAIN__: + - ws.__CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + ws.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/ws.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/ws.__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench2_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench2_configuration.sls new file mode 100644 index 0000000000..5aa6342866 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench2_configuration.sls @@ -0,0 +1,18 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + workbench2.__CLUSTER__.__DOMAIN__: + - workbench2.__CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + workbench2.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/workbench2.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/workbench2.__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench_configuration.sls new file mode 100644 index 0000000000..4620f79e37 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/letsencrypt_workbench_configuration.sls @@ -0,0 +1,18 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### LETSENCRYPT +letsencrypt: + domainsets: + workbench.__CLUSTER__.__DOMAIN__: + - workbench.__CLUSTER__.__DOMAIN__ + +### NGINX +nginx: + ### SNIPPETS + snippets: + workbench.__CLUSTER__.__DOMAIN___letsencrypt_cert.conf: + - ssl_certificate: /etc/letsencrypt/live/workbench.__CLUSTER__.__DOMAIN__/fullchain.pem + - ssl_certificate_key: /etc/letsencrypt/live/workbench.__CLUSTER__.__DOMAIN__/privkey.pem diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/locale.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/locale.sls new file mode 100644 index 0000000000..17f53a2881 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/locale.sls @@ -0,0 +1,14 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +locale: + present: + - "en_US.UTF-8 UTF-8" + default: + # Note: On debian systems don't write the second 'UTF-8' here or you will + # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8 + # Restart the minion after you corrected this! + name: 'en_US.UTF-8' + requires: 'en_US.UTF-8 UTF-8' diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_api_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_api_configuration.sls new file mode 100644 index 0000000000..c0b087045e --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_api_configuration.sls @@ -0,0 +1,28 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### ARVADOS +arvados: + config: + group: www-data + +### NGINX +nginx: + ### SITES + servers: + managed: + arvados_api: + enabled: true + overwrite: true + config: + - server: + - listen: 'localhost:8004' + - server_name: api + - root: /var/www/arvados-api/current/public + - index: index.html index.htm + - access_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.access.log combined + - error_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.error.log + - passenger_enabled: 'on' + - client_max_body_size: 128m diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls new file mode 100644 index 0000000000..3be1696602 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_controller_configuration.sls @@ -0,0 +1,61 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + 'geo $external_client': + default: 1 + '127.0.0.0/8': 0 + '__CLUSTER_INT_CIDR__': 0 + upstream controller_upstream: + - server: 'localhost:8003 fail_timeout=10s' + + ### SITES + servers: + managed: + ### DEFAULT + arvados_controller_default: + enabled: true + overwrite: true + config: + - server: + - server_name: __CLUSTER__.__DOMAIN__ + - listen: + - 80 default + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + arvados_controller_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-__CLUSTER__.__DOMAIN__-__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: __CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://controller_upstream' + - proxy_read_timeout: 300 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_set_header: 'X-External-Client $external_client' + - include: snippets/ssl_hardening_default.conf + - include: snippets/__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/controller.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/controller.__CLUSTER__.__DOMAIN__.error.log + - client_max_body_size: 128m diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepproxy_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepproxy_configuration.sls new file mode 100644 index 0000000000..5d8b37e595 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepproxy_configuration.sls @@ -0,0 +1,59 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + upstream keepproxy_upstream: + - server: 'localhost:25107 fail_timeout=10s' + + servers: + managed: + ### DEFAULT + arvados_keepproxy_default: + enabled: true + overwrite: true + config: + - server: + - server_name: keep.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + arvados_keepproxy_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-keep.__CLUSTER__.__DOMAIN__-keep.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: keep.__CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://keepproxy_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_body_buffer_size: 64M + - client_max_body_size: 64M + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: snippets/ssl_hardening_default.conf + - include: snippets/keep.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepweb_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepweb_configuration.sls new file mode 100644 index 0000000000..fca4216076 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_keepweb_configuration.sls @@ -0,0 +1,89 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + upstream collections_downloads_upstream: + - server: 'localhost:9002 fail_timeout=10s' + + servers: + managed: + ### DEFAULT + arvados_collections_download_default: + enabled: true + overwrite: true + config: + - server: + - server_name: '~^((.*--)?collections|download)\.__CLUSTER__\.__DOMAIN__' + - listen: + - 80 + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + ### COLLECTIONS + arvados_collections_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-collections.__CLUSTER__.__DOMAIN__-collections.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: '~^(.*--)?collections\.__CLUSTER__\.__DOMAIN__' + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://collections_downloads_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_max_body_size: 0 + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: snippets/ssl_hardening_default.conf + - include: snippets/collections.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/collections.__CLUSTER__.__DOMAIN__.error.log + + ### DOWNLOAD + arvados_download_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-download.__CLUSTER__.__DOMAIN__-download.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: download.__CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://collections_downloads_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_max_body_size: 0 + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: snippets/ssl_hardening_default.conf + - include: snippets/download.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/download.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/download.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls new file mode 100644 index 0000000000..a2df3ff096 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls @@ -0,0 +1,53 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + install_from_phusionpassenger: true + lookup: + passenger_package: libnginx-mod-http-passenger + passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf + + ### SNIPPETS + snippets: + # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4 + ssl_hardening_default.conf: + - ssl_session_timeout: 1d + - ssl_session_cache: 'shared:arvadosSSL:10m' + - ssl_session_tickets: 'off' + + # intermediate configuration + - ssl_protocols: TLSv1.2 TLSv1.3 + - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + - ssl_prefer_server_ciphers: 'off' + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + - add_header: 'Strict-Transport-Security "max-age=63072000" always' + + # OCSP stapling + - ssl_stapling: 'on' + - ssl_stapling_verify: 'on' + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates + + # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam + # - ssl_dhparam: /path/to/dhparam + + # replace with the IP address of your resolver + # - resolver: 127.0.0.1 + + ### SERVER + server: + config: + include: 'modules-enabled/*.conf' + worker_processes: 4 + + ### SITES + servers: + managed: + # Remove default webserver + default: + enabled: false diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_webshell_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_webshell_configuration.sls new file mode 100644 index 0000000000..46f8ad0386 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_webshell_configuration.sls @@ -0,0 +1,76 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + + ### STREAMS + http: + upstream webshell_upstream: + - server: 'localhost:4200 fail_timeout=10s' + + ### SITES + servers: + managed: + arvados_webshell_default: + enabled: true + overwrite: true + config: + - server: + - server_name: webshell.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + arvados_webshell_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-webshell.__CLUSTER__.__DOMAIN__-webshell.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: webshell.__CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /shell.__CLUSTER__.__DOMAIN__: + - proxy_pass: 'http://webshell_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_ssl_session_reuse: 'off' + + - "if ($request_method = 'OPTIONS')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + - add_header: "'Access-Control-Max-Age' 1728000" + - add_header: "'Content-Type' 'text/plain charset=UTF-8'" + - add_header: "'Content-Length' 0" + - return: 204 + + - "if ($request_method = 'POST')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + + - "if ($request_method = 'GET')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + + - include: snippets/ssl_hardening_default.conf + - include: snippets/webshell.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log + diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_websocket_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_websocket_configuration.sls new file mode 100644 index 0000000000..e89b780da6 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_websocket_configuration.sls @@ -0,0 +1,60 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### NGINX +nginx: + ### SERVER + server: + config: + ### STREAMS + http: + upstream websocket_upstream: + - server: 'localhost:8005 fail_timeout=10s' + + servers: + managed: + ### DEFAULT + arvados_websocket_default: + enabled: true + overwrite: true + config: + - server: + - server_name: ws.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + arvados_websocket_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-ws.__CLUSTER__.__DOMAIN__-ws.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: ws.__CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://websocket_upstream' + - proxy_read_timeout: 600 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: 'Host $host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'Upgrade $http_upgrade' + - proxy_set_header: 'Connection "upgrade"' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_buffering: 'off' + - client_body_buffer_size: 64M + - client_max_body_size: 64M + - proxy_http_version: '1.1' + - proxy_request_buffering: 'off' + - include: snippets/ssl_hardening_default.conf + - include: snippets/ws.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench2_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench2_configuration.sls new file mode 100644 index 0000000000..a3e58e2e25 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench2_configuration.sls @@ -0,0 +1,50 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### ARVADOS +arvados: + config: + group: www-data + +### NGINX +nginx: + ### SITES + servers: + managed: + ### DEFAULT + arvados_workbench2_default: + enabled: true + overwrite: true + config: + - server: + - server_name: workbench2.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + arvados_workbench2_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-workbench2.__CLUSTER__.__DOMAIN__-workbench2.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: workbench2.__CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - root: /var/www/arvados-workbench2/workbench2 + - try_files: '$uri $uri/ /index.html' + - 'if (-f $document_root/maintenance.html)': + - return: 503 + - location /config.json: + - return: {{ "200 '" ~ '{"API_HOST":"__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }} + - include: snippets/ssl_hardening_default.conf + - include: snippets/workbench2.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench_configuration.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench_configuration.sls new file mode 100644 index 0000000000..38e59cc1ba --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_workbench_configuration.sls @@ -0,0 +1,75 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### ARVADOS +arvados: + config: + group: www-data + +### NGINX +nginx: + ### SERVER + server: + config: + + ### STREAMS + http: + upstream workbench_upstream: + - server: 'localhost:9000 fail_timeout=10s' + + ### SITES + servers: + managed: + ### DEFAULT + arvados_workbench_default: + enabled: true + overwrite: true + config: + - server: + - server_name: workbench.__CLUSTER__.__DOMAIN__ + - listen: + - 80 + - include: snippets/letsencrypt_well_known.conf + - location /: + - return: '301 https://$host$request_uri' + + arvados_workbench_ssl: + enabled: true + overwrite: true + requires: + cmd: create-initial-cert-workbench.__CLUSTER__.__DOMAIN__-workbench.__CLUSTER__.__DOMAIN__ + config: + - server: + - server_name: workbench.__CLUSTER__.__DOMAIN__ + - listen: + - __CONTROLLER_EXT_SSL_PORT__ http2 ssl + - index: index.html index.htm + - location /: + - proxy_pass: 'http://workbench_upstream' + - proxy_read_timeout: 300 + - proxy_connect_timeout: 90 + - proxy_redirect: 'off' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - include: snippets/ssl_hardening_default.conf + - include: snippets/workbench.__CLUSTER__.__DOMAIN___letsencrypt_cert[.]conf + - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined + - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log + + arvados_workbench_upstream: + enabled: true + overwrite: true + config: + - server: + - listen: 'localhost:9000' + - server_name: workbench + - root: /var/www/arvados-workbench/current/public + - index: index.html index.htm + - passenger_enabled: 'on' + # yamllint disable-line rule:line-length + - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.access.log combined + - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.error.log diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls new file mode 100644 index 0000000000..a0da9a1c05 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls @@ -0,0 +1,42 @@ +--- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +### POSTGRESQL +postgres: + use_upstream_repo: true + version: '11' + postgresconf: |- + listen_addresses = '*' # listen on all interfaces + acls: + - ['local', 'all', 'postgres', 'peer'] + - ['local', 'all', 'all', 'peer'] + - ['host', 'all', 'all', '127.0.0.1/32', 'md5'] + - ['host', 'all', 'all', '::1/128', 'md5'] + - ['host', '__CLUSTER___arvados', '__CLUSTER___arvados', '127.0.0.1/32'] + - ['host', '__CLUSTER___arvados', '__CLUSTER___arvados', '__CONTROLLER_INT_IP__/32'] + users: + __CLUSTER___arvados: + ensure: present + password: __DATABASE_PASSWORD__ + + # tablespaces: + # arvados_tablespace: + # directory: /path/to/some/tbspace/arvados_tbsp + # owner: arvados + + databases: + __CLUSTER___arvados: + owner: __CLUSTER___arvados + template: template0 + lc_ctype: en_US.utf8 + lc_collate: en_US.utf8 + # tablespace: arvados_tablespace + schemas: + public: + owner: __CLUSTER___arvados + extensions: + pg_trgm: + if_not_exists: true + schema: public diff --git a/tools/salt-install/config_examples/multi_host/aws/states/host_entries.sls b/tools/salt-install/config_examples/multi_host/aws/states/host_entries.sls new file mode 100644 index 0000000000..82fb6f4ec9 --- /dev/null +++ b/tools/salt-install/config_examples/multi_host/aws/states/host_entries.sls @@ -0,0 +1,71 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +{%- set curr_tpldir = tpldir %} +{%- set tpldir = 'arvados' %} +{%- from "arvados/map.jinja" import arvados with context %} +{%- set tpldir = curr_tpldir %} + +#CRUDE, but functional +extra_extra_hosts_entries_etc_hosts_database_host_present: + host.present: + - ip: __DATABASE_INT_IP__ + - names: + - db.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + - database.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_api_host_present: + host.present: + - ip: __CONTROLLER_INT_IP__ + - names: + - {{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_websocket_host_present: + host.present: + - ip: __CONTROLLER_INT_IP__ + - names: + - ws.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_workbench_host_present: + host.present: + - ip: __WORKBENCH1_INT_IP__ + - names: + - workbench.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_workbench2_host_present: + host.present: + - ip: __WORKBENCH1_INT_IP__ + - names: + - workbench2.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_keepproxy_host_present: + host.present: + - ip: __KEEP_INT_IP__ + - names: + - keep.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_keepweb_host_present: + host.present: + - ip: __KEEP_INT_IP__ + - names: + - download.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + - collections.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_shell_host_present: + host.present: + - ip: __WEBSHELL_INT_IP__ + - names: + - shell.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_keep0_host_present: + host.present: + - ip: __KEEPSTORE0_INT_IP__ + - names: + - keep0.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} + +extra_extra_hosts_entries_etc_hosts_keep1_host_present: + host.present: + - ip: __KEEPSTORE1_INT_IP__ + - names: + - keep1.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} diff --git a/tools/terraform/.gitignore b/tools/terraform/.gitignore new file mode 100644 index 0000000000..df47a74b5b --- /dev/null +++ b/tools/terraform/.gitignore @@ -0,0 +1,7 @@ +.DS_Store +.terraform +examples +*backup +*disabled +.terraform.lock.hcl +terraform.tfstate* -- 2.30.2