feat(provision): refactor to add other setup examples
authorJavier Bértoli <jbertoli@curii.com>
Tue, 26 Jan 2021 11:58:07 +0000 (08:58 -0300)
committerJavier Bértoli <jbertoli@curii.com>
Tue, 16 Feb 2021 13:51:28 +0000 (10:51 -0300)
refs #17246
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli@curii.com>

30 files changed:
tools/salt-install/config_examples/single_host/multiple_hostnames/README.md [new file with mode: 0644]
tools/salt-install/config_examples/single_host/multiple_hostnames/arvados.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/multiple_hostnames/docker.sls [moved from tools/salt-install/single_host/docker.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/locale.sls [moved from tools/salt-install/single_host/locale.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_api_configuration.sls [moved from tools/salt-install/single_host/nginx_api_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_controller_configuration.sls [moved from tools/salt-install/single_host/nginx_controller_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_keepproxy_configuration.sls [moved from tools/salt-install/single_host/nginx_keepproxy_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_keepweb_configuration.sls [moved from tools/salt-install/single_host/nginx_keepweb_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_passenger.sls [moved from tools/salt-install/single_host/nginx_passenger.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_webshell_configuration.sls [moved from tools/salt-install/single_host/nginx_webshell_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_websocket_configuration.sls [moved from tools/salt-install/single_host/nginx_websocket_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_workbench2_configuration.sls [moved from tools/salt-install/single_host/nginx_workbench2_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/nginx_workbench_configuration.sls [moved from tools/salt-install/single_host/nginx_workbench_configuration.sls with 100% similarity]
tools/salt-install/config_examples/single_host/multiple_hostnames/postgresql.sls [moved from tools/salt-install/single_host/postgresql.sls with 100% similarity]
tools/salt-install/config_examples/single_host/single_hostname/README.md [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/arvados.sls [moved from tools/salt-install/single_host/arvados.sls with 78% similarity]
tools/salt-install/config_examples/single_host/single_hostname/docker.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/locale.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_api_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_controller_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_keepproxy_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_keepweb_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_passenger.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_webshell_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_websocket_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_workbench2_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/nginx_workbench_configuration.sls [new file with mode: 0644]
tools/salt-install/config_examples/single_host/single_hostname/postgresql.sls [new file with mode: 0644]
tools/salt-install/local.params.example [new file with mode: 0644]
tools/salt-install/provision.sh

diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/README.md b/tools/salt-install/config_examples/single_host/multiple_hostnames/README.md
new file mode 100644 (file)
index 0000000..17ca89a
--- /dev/null
@@ -0,0 +1,20 @@
+Single host with multiple hostnames
+===================================
+
+These files let you setup Arvados on a single host using different hostnames
+for each of its components nginx's virtualhosts.
+
+The hostnames are composed after the variables "CLUSTER" and "DOMAIN" set in
+the `local.params` file.
+
+The virtual hosts' hostnames that will be used are:
+
+* CLUSTER.DOMAIN
+* collections.CLUSTER.DOMAIN
+* download.CLUSTER.DOMAIN
+* keep.CLUSTER.DOMAIN
+* keep0.CLUSTER.DOMAIN
+* webshell.CLUSTER.DOMAIN
+* workbench.CLUSTER.DOMAIN
+* workbench2.CLUSTER.DOMAIN
+* ws.CLUSTER.DOMAIN
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/arvados.sls
new file mode 100644 (file)
index 0000000..4aa4735
--- /dev/null
@@ -0,0 +1,159 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# The variables commented out are the default values that the formula uses.
+# The uncommented values are REQUIRED values. If you don't set them, running
+# this formula will fail.
+arvados:
+  ### GENERAL CONFIG
+  version: '__VERSION__'
+  ## It makes little sense to disable this flag, but you can, if you want :)
+  # use_upstream_repo: true
+
+  ## Repo URL is built with grains values. If desired, it can be completely
+  ## overwritten with the pillar parameter 'repo_url'
+  # repo:
+  #   humanname: Arvados Official Repository
+
+  release: __RELEASE__
+
+  ## IMPORTANT!!!!!
+  ## api, workbench and shell require some gems, so you need to make sure ruby
+  ## and deps are installed in order to install and compile the gems.
+  ## We default to `false` in these two variables as it's expected you already
+  ## manage OS packages with some other tool and you don't want us messing up
+  ## with your setup.
+  ruby:
+    ## We set these to `true` here for testing purposes.
+    ## They both default to `false`.
+    manage_ruby: true
+    manage_gems_deps: true
+    # pkg: ruby
+    # gems_deps:
+    #     - curl
+    #     - g++
+    #     - gcc
+    #     - git
+    #     - libcurl4
+    #     - libcurl4-gnutls-dev
+    #     - libpq-dev
+    #     - libxml2
+    #     - libxml2-dev
+    #     - make
+    #     - python3-dev
+    #     - ruby-dev
+    #     - zlib1g-dev
+
+  # config:
+  #   file: /etc/arvados/config.yml
+  #   user: root
+  ## IMPORTANT!!!!!
+  ## If you're intalling any of the rails apps (api, workbench), the group
+  ## should be set to that of the web server, usually `www-data`
+  #   group: root
+  #   mode: 640
+
+  ### ARVADOS CLUSTER CONFIG
+  cluster:
+    name: __CLUSTER__
+    domain: __DOMAIN__
+
+    database:
+      # max concurrent connections per arvados server daemon
+      # connection_pool_max: 32
+      name: arvados
+      host: 127.0.0.1
+      password: changeme_arvados
+      user: arvados
+      encoding: en_US.utf8
+      client_encoding: UTF8
+
+    tls:
+      # certificate: ''
+      # key: ''
+      # required to test with arvados-snakeoil certs
+      insecure: true
+
+    ### TOKENS
+    tokens:
+      system_root: changemesystemroottoken
+      management: changememanagementtoken
+      rails_secret: changemerailssecrettoken
+      anonymous_user: changemeanonymoususertoken
+
+    ### KEYS
+    secrets:
+      blob_signing_key: changemeblobsigningkey
+      workbench_secret_key: changemeworkbenchsecretkey
+      dispatcher_access_key: changemedispatcheraccesskey
+      dispatcher_secret_key: changeme_dispatchersecretkey
+      keep_access_key: changemekeepaccesskey
+      keep_secret_key: changemekeepsecretkey
+
+    Login:
+      Test:
+        Enable: true
+        Users:
+          __INITIAL_USER__:
+            Email: __INITIAL_USER_EMAIL__
+            Password: __INITIAL_USER_PASSWORD__
+
+    ### VOLUMES
+    ## This should usually match all your `keepstore` instances
+    Volumes:
+      # the volume name will be composed with
+      # <cluster>-nyw5e-<volume>
+      __CLUSTER__-nyw5e-000000000000000:
+        AccessViaHosts:
+          'http://keep0.__CLUSTER__.__DOMAIN__:25107':
+            ReadOnly: false
+        Replication: 2
+        Driver: Directory
+        DriverParameters:
+          Root: /tmp
+
+    Users:
+      NewUsersAreActive: true
+      AutoAdminFirstUser: true
+      AutoSetupNewUsers: true
+      AutoSetupNewUsersWithRepository: true
+
+    Services:
+      Controller:
+        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
+        InternalURLs:
+          'http://controller.internal:8003': {}
+      DispatchCloud:
+        InternalURLs:
+          'http://__CLUSTER__.__DOMAIN__:9006': {}
+      Keepbalance:
+        InternalURLs:
+          'http://__CLUSTER__.__DOMAIN__:9005': {}
+      Keepproxy:
+        ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
+        InternalURLs:
+          'http://keep.internal:25100': {}
+      Keepstore:
+        InternalURLs:
+          'http://keep0.__CLUSTER__.__DOMAIN__:25107': {}
+      RailsAPI:
+        InternalURLs:
+          'http://api.internal:8004': {}
+      WebDAV:
+        ExternalURL: 'https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
+        InternalURLs:
+          'http://collections.internal:9002': {}
+      WebDAVDownload:
+        ExternalURL: 'https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
+      WebShell:
+        ExternalURL: 'https://webshell.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
+      Websocket:
+        ExternalURL: 'wss://ws.__CLUSTER__.__DOMAIN__/websocket'
+        InternalURLs:
+          'http://ws.internal:8005': {}
+      Workbench1:
+        ExternalURL: 'https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
+      Workbench2:
+        ExternalURL: 'https://workbench2.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__'
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/README.md b/tools/salt-install/config_examples/single_host/single_hostname/README.md
new file mode 100644 (file)
index 0000000..9c7ab96
--- /dev/null
@@ -0,0 +1,23 @@
+Single host with a single hostname
+==================================
+
+These files let you setup Arvados on a single host using a single hostname
+for all of its components nginx's virtualhosts.
+
+The hostname MUST be given in the `local.params` file. The script won't try
+to guess it because, depending on the network architecture where you're
+installing Arvados, things might not work as expected.
+
+The services will be available on the same hostname but different ports,
+which can be given on the `local.params` file or will default to the following
+values:
+
+* CLUSTER.DOMAIN
+* collections
+* download
+* keep
+* keep0
+* webshell
+* workbench
+* workbench2
+* ws
similarity index 78%
rename from tools/salt-install/single_host/arvados.sls
rename to tools/salt-install/config_examples/single_host/single_hostname/arvados.sls
index a06244270c237150f159220fffaab4de1a9f2f19..e5e458665778c8fc714661b52099f8e975c5d5cb 100644 (file)
@@ -107,7 +107,7 @@ arvados:
       # <cluster>-nyw5e-<volume>
       __CLUSTER__-nyw5e-000000000000000:
         AccessViaHosts:
-          http://keep0.__CLUSTER__.__DOMAIN__:25107:
+          'http://__HOSTNAME__:25107':
             ReadOnly: false
         Replication: 2
         Driver: Directory
@@ -122,38 +122,38 @@ arvados:
 
     Services:
       Controller:
-        ExternalURL: https://__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__CONTROLLER_EXT_SSL_PORT__'
         InternalURLs:
-          http://controller.internal:8003: {}
+          'http://controller.internal:8003': {}
       DispatchCloud:
         InternalURLs:
-          http://__CLUSTER__.__DOMAIN__:9006: {}
+          'http://__HOSTNAME__:9006': {}
       Keepbalance:
         InternalURLs:
-          http://__CLUSTER__.__DOMAIN__:9005: {}
+          'http://__HOSTNAME__:9005': {}
       Keepproxy:
-        ExternalURL: https://keep.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__KEEP_EXT_SSL_PORT__'
         InternalURLs:
-          http://keep.internal:25100: {}
+          'http://keep.internal:25100': {}
       Keepstore:
         InternalURLs:
-          http://keep0.__CLUSTER__.__DOMAIN__:25107: {}
+          'http://keep0.internal:25107': {}
       RailsAPI:
         InternalURLs:
-          http://api.internal:8004: {}
+          'http://api.internal:8004': {}
       WebDAV:
-        ExternalURL: https://collections.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__KEEPWEB_EXT_SSL_PORT__'
         InternalURLs:
-          http://collections.internal:9002: {}
+          'http://collections.internal:9002': {}
       WebDAVDownload:
-        ExternalURL: https://download.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__KEEPWEB_EXT_SSL_PORT__'
       WebShell:
-        ExternalURL: https://webshell.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__WEBSHELL_EXT_SSL_PORT__'
       Websocket:
-        ExternalURL: wss://ws.__CLUSTER__.__DOMAIN__/websocket
+        ExternalURL: 'wss://__HOSTNAME__:__WEBSOCKET_EXT_SSL_PORT__/websocket'
         InternalURLs:
-          http://ws.internal:8005: {}
+          'http://ws.internal:8005': {}
       Workbench1:
-        ExternalURL: https://workbench.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__WORKBENCH1_EXT_SSL_PORT__'
       Workbench2:
-        ExternalURL: https://workbench2.__CLUSTER__.__DOMAIN__:__HOST_SSL_PORT__
+        ExternalURL: 'https://__HOSTNAME__:__WORKBENCH2_EXT_SSL_PORT__'
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/docker.sls b/tools/salt-install/config_examples/single_host/single_hostname/docker.sls
new file mode 100644 (file)
index 0000000..54d2256
--- /dev/null
@@ -0,0 +1,9 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+docker:
+  pkg:
+    docker:
+      use_upstream: package
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/locale.sls b/tools/salt-install/config_examples/single_host/single_hostname/locale.sls
new file mode 100644 (file)
index 0000000..17f53a2
--- /dev/null
@@ -0,0 +1,14 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+locale:
+  present:
+    - "en_US.UTF-8 UTF-8"
+  default:
+    # Note: On debian systems don't write the second 'UTF-8' here or you will
+    # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8
+    # Restart the minion after you corrected this!
+    name: 'en_US.UTF-8'
+    requires: 'en_US.UTF-8 UTF-8'
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_api_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_api_configuration.sls
new file mode 100644 (file)
index 0000000..b2f12c7
--- /dev/null
@@ -0,0 +1,28 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+  config:
+    group: www-data
+
+### NGINX
+nginx:
+  ### SITES
+  servers:
+    managed:
+      arvados_api:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - listen: 'api.internal:8004'
+            - server_name: api
+            - root: /var/www/arvados-api/current/public
+            - index:  index.html index.htm
+            - access_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+            - error_log: /var/log/nginx/api.__CLUSTER__.__DOMAIN__-upstream.error.log
+            - passenger_enabled: 'on'
+            - client_max_body_size: 128m
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_controller_configuration.sls
new file mode 100644 (file)
index 0000000..2eb33b8
--- /dev/null
@@ -0,0 +1,58 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        'geo $external_client':
+          default: 1
+          '127.0.0.0/8': 0
+        upstream controller_upstream:
+          - server: 'controller.internal:8003  fail_timeout=10s'
+
+  ### SITES
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_controller_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - 80 default
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_controller_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __CONTROLLER_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://controller_upstream'
+              - proxy_read_timeout: 300
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_set_header: 'X-External-Client $external_client'
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
+            - client_max_body_size: 128m
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_keepproxy_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_keepproxy_configuration.sls
new file mode 100644 (file)
index 0000000..b26de27
--- /dev/null
@@ -0,0 +1,57 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        upstream keepproxy_upstream:
+          - server: 'keep.internal:25100 fail_timeout=10s'
+
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_keepproxy_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __KEEP_EXT_SSL_PORT__
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_keepproxy_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __KEEP_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://keepproxy_upstream'
+              - proxy_read_timeout: 90
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_buffering: 'off'
+            - client_body_buffer_size: 64M
+            - client_max_body_size: 64M
+            - proxy_http_version: '1.1'
+            - proxy_request_buffering: 'off'
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_keepweb_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_keepweb_configuration.sls
new file mode 100644 (file)
index 0000000..98a3cdf
--- /dev/null
@@ -0,0 +1,57 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        upstream collections_downloads_upstream:
+          - server: 'collections.internal:9002 fail_timeout=10s'
+
+  servers:
+    managed:
+      ### COLLECTIONS / DOWNLOAD
+      arvados_collections_download_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __KEEPWEB_EXT_SSL_PORT__
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      ### COLLECTIONS / DOWNLOAD
+      arvados_collections_download_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __KEEPWEB_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://collections_downloads_upstream'
+              - proxy_read_timeout: 90
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_buffering: 'off'
+            - client_max_body_size: 0
+            - proxy_http_version: '1.1'
+            - proxy_request_buffering: 'off'
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/keepweb.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/keepweb.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_passenger.sls
new file mode 100644 (file)
index 0000000..6ce75fa
--- /dev/null
@@ -0,0 +1,24 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  install_from_phusionpassenger: true
+  lookup:
+    passenger_package: libnginx-mod-http-passenger
+    passenger_config_file: /etc/nginx/conf.d/mod-http-passenger.conf
+
+  ### SERVER
+  server:
+    config:
+      include: 'modules-enabled/*.conf'
+      worker_processes: 4
+
+  ### SITES
+  servers:
+    managed:
+      # Remove default webserver
+      default:
+        enabled: false
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_webshell_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_webshell_configuration.sls
new file mode 100644 (file)
index 0000000..dac6061
--- /dev/null
@@ -0,0 +1,74 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+
+      ### STREAMS
+      http:
+        upstream webshell_upstream:
+          - server: 'shell.internal:4200 fail_timeout=10s'
+
+  ### SITES
+  servers:
+    managed:
+      arvados_webshell_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __WEBSHELL_EXT_SSL_PORT__
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_webshell_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __WEBSHELL_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /__HOSTNAME__:
+              - proxy_pass: 'http://webshell_upstream'
+              - proxy_read_timeout: 90
+              - proxy_connect_timeout: 90
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_ssl_session_reuse: 'off'
+
+              - "if ($request_method = 'OPTIONS')":
+                - add_header: "'Access-Control-Allow-Origin' '*'"
+                - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+                - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+                - add_header: "'Access-Control-Max-Age' 1728000"
+                - add_header: "'Content-Type' 'text/plain charset=UTF-8'"
+                - add_header: "'Content-Length' 0"
+                - return: 204
+
+              - "if ($request_method = 'POST')":
+                - add_header: "'Access-Control-Allow-Origin' '*'"
+                - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+                - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+              - "if ($request_method = 'GET')":
+                - add_header: "'Access-Control-Allow-Origin' '*'"
+                - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
+                - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
+
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
+
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_websocket_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_websocket_configuration.sls
new file mode 100644 (file)
index 0000000..827524c
--- /dev/null
@@ -0,0 +1,58 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+      ### STREAMS
+      http:
+        upstream websocket_upstream:
+          - server: 'ws.internal:8005 fail_timeout=10s'
+
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_websocket_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __WEBSOCKET_EXT_SSL_PORT__
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_websocket_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __WEBSOCKET_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://websocket_upstream'
+              - proxy_read_timeout: 600
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: 'Host $host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'Upgrade $http_upgrade'
+              - proxy_set_header: 'Connection "upgrade"'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+              - proxy_buffering: 'off'
+            - client_body_buffer_size: 64M
+            - client_max_body_size: 64M
+            - proxy_http_version: '1.1'
+            - proxy_request_buffering: 'off'
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_workbench2_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_workbench2_configuration.sls
new file mode 100644 (file)
index 0000000..7f90cbc
--- /dev/null
@@ -0,0 +1,48 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+  config:
+    group: www-data
+
+### NGINX
+nginx:
+  ### SITES
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_workbench2_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __WORKBENCH2_EXT_SSL_PORT__
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_workbench2_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench2.__HOSTNAME__
+            - listen:
+              - __WORKBENCH2_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - root: /var/www/arvados-workbench2/workbench2
+              - try_files: '$uri $uri/ /index.html'
+              - 'if (-f $document_root/maintenance.html)':
+                - return: 503
+            - location /config.json:
+              - return: {{ "200 '" ~ '{"API_HOST":"__HOSTNAME__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }}
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/nginx_workbench_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/nginx_workbench_configuration.sls
new file mode 100644 (file)
index 0000000..0cbd3e1
--- /dev/null
@@ -0,0 +1,73 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### ARVADOS
+arvados:
+  config:
+    group: www-data
+
+### NGINX
+nginx:
+  ### SERVER
+  server:
+    config:
+
+      ### STREAMS
+      http:
+        upstream workbench_upstream:
+          - server: 'workbench.internal:9000 fail_timeout=10s'
+
+  ### SITES
+  servers:
+    managed:
+      ### DEFAULT
+      arvados_workbench_default:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: __HOSTNAME__
+            - listen:
+              - __WORKBENCH_EXT_SSL_PORT__
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_workbench_ssl:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench.__HOSTNAME__
+            - listen:
+              - __WORKBENCH1_EXT_SSL_PORT__ http2 ssl
+            - index: index.html index.htm
+            - location /:
+              - proxy_pass: 'http://workbench_upstream'
+              - proxy_read_timeout: 300
+              - proxy_connect_timeout: 90
+              - proxy_redirect: 'off'
+              - proxy_set_header: X-Forwarded-Proto https
+              - proxy_set_header: 'Host $http_host'
+              - proxy_set_header: 'X-Real-IP $remote_addr'
+              - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
+            - include: 'snippets/arvados-snakeoil.conf'
+            - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
+            - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
+
+      arvados_workbench_upstream:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - listen: 'workbench.internal:9000'
+            - server_name: workbench
+            - root: /var/www/arvados-workbench/current/public
+            - index:  index.html index.htm
+            - passenger_enabled: 'on'
+            # yamllint disable-line rule:line-length
+            - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.access.log combined
+            - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__-upstream.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/postgresql.sls b/tools/salt-install/config_examples/single_host/single_hostname/postgresql.sls
new file mode 100644 (file)
index 0000000..56b0a42
--- /dev/null
@@ -0,0 +1,42 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### POSTGRESQL
+postgres:
+  use_upstream_repo: false
+  pkgs_extra:
+    - postgresql-contrib
+  postgresconf: |-
+    listen_addresses = '*'  # listen on all interfaces
+  acls:
+    - ['local', 'all', 'postgres', 'peer']
+    - ['local', 'all', 'all', 'peer']
+    - ['host', 'all', 'all', '127.0.0.1/32', 'md5']
+    - ['host', 'all', 'all', '::1/128', 'md5']
+    - ['host', 'arvados', 'arvados', '127.0.0.1/32']
+  users:
+    arvados:
+      ensure: present
+      password: changeme_arvados
+
+  # tablespaces:
+  #   arvados_tablespace:
+  #     directory: /path/to/some/tbspace/arvados_tbsp
+  #     owner: arvados
+
+  databases:
+    arvados:
+      owner: arvados
+      template: template0
+      lc_ctype: en_US.utf8
+      lc_collate: en_US.utf8
+      # tablespace: arvados_tablespace
+      schemas:
+        public:
+          owner: arvados
+      extensions:
+        pg_trgm:
+          if_not_exists: true
+          schema: public
diff --git a/tools/salt-install/local.params.example b/tools/salt-install/local.params.example
new file mode 100644 (file)
index 0000000..a88301b
--- /dev/null
@@ -0,0 +1,64 @@
+##########################################################
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+# These are the basic parameters to configure the installation
+
+# The 5 letters name you want to give your cluster
+CLUSTER="fixme"
+
+# The domainname you want tou give to your cluster's hosts
+DOMAIN="some.domain"
+
+# When setting the cluster in a single host, you can use a single hostname
+# to access all the instances. When using virtualization (ie AWS), this should be
+# the EXTERNAL hostname for the instance.
+# If empty, the INTERNAL HOST IP will be used
+HOSTNAME=""
+CONTROLLER_EXT_SSL_PORT=8000
+KEEP_EXT_SSL_PORT=25101
+# Both for collections and downloads
+KEEPWEB_EXT_SSL_PORT=9002
+WEBSHELL_EXT_SSL_PORT=4202
+WEBSOCKET_EXT_SSL_PORT=8002
+WORKBENCH1_EXT_SSL_PORT=443
+WORKBENCH2_EXT_SSL_PORT=3001
+
+INITIAL_USER="admin"
+
+# If not specified, the initial user email will be composed as
+# INITIAL_USER@CLUSTER.DOMAIN
+INITIAL_USER_EMAIL="admin@fixme.localdomain"
+INITIAL_USER_PASSWORD="password"
+
+# The example config files you want to use. There are a few examples
+# under 'config_examples' 
+CONFIG_DIR="config_examples/single_host/single_hostname"
+
+# Which release of Arvados repo you want to use
+RELEASE="production"
+# Which version of Arvados you want to install. Defaults to 'latest'
+# in the desired repo
+VERSION="latest"
+
+# Host SSL port where you want to point your browser to access Arvados
+# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
+# You can point it to another port if desired
+# In Vagrant, make sure it matches what you set in the Vagrantfile
+HOST_SSL_PORT=443
+
+# This is an arvados-formula setting.
+# If branch is set, the script will switch to it before running salt
+# Usually not needed, only used for testing
+BRANCH="master"
+
+##########################################################
+# Usually there's no need to modify things below this line
+
+# Formulas versions
+ARVADOS_TAG="v1.1.4"
+POSTGRES_TAG="v0.41.3"
+NGINX_TAG="v2.4.0"
+DOCKER_TAG="v1.0.0"
+LOCALE_TAG="v0.3.4"
index b97d719658e09c3cdad2b6dda396c77649cd99c5..f3df4109a05cd88980662f4cdcbe0a592e6b3664 100755 (executable)
@@ -15,15 +15,6 @@ set -o pipefail
 # capture the directory that the script is running from
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-CONFIG_DIR="single_host"
-RELEASE="production"
-VERSION="latest"
-ARVADOS_TAG="v1.1.4"
-POSTGRES_TAG="v0.41.3"
-NGINX_TAG="v2.4.0"
-DOCKER_TAG="v1.0.0"
-LOCALE_TAG="v0.3.4"
-
 usage() {
   echo >&2
   echo >&2 "Usage: ${0} [-h] [-h]"
@@ -109,10 +100,35 @@ arguments() {
 }
 
 CONFIG="${SCRIPT_DIR}/local.params"
+CONFIG_DIR="config_examples/single_host/multiple_hostnames"
 LOG_LEVEL="info"
 HOST_SSL_PORT=443
 TESTS_DIR="tests"
 
+CLUSTER=""
+DOMAIN=""
+HOSTNAME=""
+INITIAL_USER=""
+INITIAL_USER_EMAIL=""
+INITIAL_USER_PASSWORD=""
+
+CONTROLLER_EXT_SSL_PORT=8000
+KEEP_EXT_SSL_PORT=25101
+# Both for collections and downloads
+KEEPWEB_EXT_SSL_PORT=9002
+WEBSHELL_EXT_SSL_PORT=4202
+WEBSOCKET_EXT_SSL_PORT=8002
+WORKBENCH1_EXT_SSL_PORT=443
+WORKBENCH2_EXT_SSL_PORT=3001
+
+RELEASE="production"
+VERSION="latest"
+ARVADOS_TAG="v1.1.4"
+POSTGRES_TAG="v0.41.3"
+NGINX_TAG="v2.4.0"
+DOCKER_TAG="v1.0.0"
+LOCALE_TAG="v0.3.4"
+
 arguments ${@}
 
 if [ -s ${CONFIG} ]; then
@@ -156,9 +172,7 @@ pillar_roots:
     - ${P_DIR}
 EOFSM
 
-mkdir -p ${S_DIR}
-mkdir -p ${F_DIR}
-mkdir -p ${P_DIR}
+mkdir -p ${S_DIR} ${F_DIR} ${P_DIR}
 
 # States
 cat > ${S_DIR}/top.sls << EOFTSLS
@@ -227,8 +241,15 @@ for f in "${SOURCE_PILLARS_DIR}"/*; do
   sed "s/__CLUSTER__/${CLUSTER}/g;
        s/__DOMAIN__/${DOMAIN}/g;
        s/__RELEASE__/${RELEASE}/g;
+       s/__CONTROLLER_EXT_SSL_PORT__/${CONTROLLER_EXT_SSL_PORT}/g;
+       s/__KEEP_EXT_SSL_PORT__/${KEEP_EXT_SSL_PORT}/g;
+       s/__WEBSHELL_EXT_SSL_PORT__/${WEBSHELL_EXT_SSL_PORT}/g;
+       s/__WORKBENCH1_EXT__SSL_PORT__/${WORKBENCH1_EXT__SSL_PORT}/g;
+       s/__WORKBENCH2_EXT__SSL_PORT__/${WORKBENCH2_EXT__SSL_PORT}/g;
+       s/__WEBSOCKET_EXT_SSL_PORT__/${WEBSOCKET_EXT_SSL_PORT}/g;
+       s/__HOSTNAME__/${HOSTNAME}/g;
+       s/__KEEPWEB_EXT_SSL_PORT__/${KEEPWEB_EXT_SSL_PORT}/g;
        s/__HOST_SSL_PORT__/${HOST_SSL_PORT}/g;
-       s/__GUEST_SSL_PORT__/${GUEST_SSL_PORT}/g;
        s/__INITIAL_USER__/${INITIAL_USER}/g;
        s/__INITIAL_USER_EMAIL__/${INITIAL_USER_EMAIL}/g;
        s/__INITIAL_USER_PASSWORD__/${INITIAL_USER_PASSWORD}/g;