19472: Syncs salt installer's main branch with commit 1e0108e1a
authorLucas Di Pentima <lucas.dipentima@curii.com>
Thu, 15 Sep 2022 20:52:31 +0000 (17:52 -0300)
committerLucas Di Pentima <lucas.dipentima@curii.com>
Thu, 15 Sep 2022 20:52:31 +0000 (17:52 -0300)
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima@curii.com>

tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
tools/salt-install/config_examples/multi_host/aws/pillars/arvados_development.sls [deleted file]
tools/salt-install/installer.sh
tools/salt-install/provision.sh
tools/salt-install/tests/run-test.sh

index 02653082f30cafb75c03d28764d7b9115aab9214..941bd95f106ec1a35859a079ca87a1fb9a6425a8 100644 (file)
@@ -1,5 +1,3 @@
-# -*- coding: utf-8 -*-
-# vim: ft=yaml
 ---
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
@@ -74,20 +72,13 @@ arvados:
       host: __DATABASE_INT_IP__
       password: "__DATABASE_PASSWORD__"
       user: __CLUSTER___arvados
-      extra_conn_params:
-        client_encoding: UTF8
-      # Centos7 does not enable SSL by default, so we disable
-      # it here just for testing of the formula purposes only.
-      # You should not do this in production, and should
-      # configure Postgres certificates correctly
-      {%- if grains.os_family in ('RedHat',) %}
-        sslmode: disable
-      {%- endif %}
+      encoding: en_US.utf8
+      client_encoding: UTF8
 
     tls:
       # certificate: ''
       # key: ''
-      # When using arvados-snakeoil certs set insecure: true
+      # required to test with arvados-snakeoil certs
       insecure: false
 
     resources:
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados_development.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados_development.sls
deleted file mode 100644 (file)
index 6163cd3..0000000
+++ /dev/null
@@ -1,182 +0,0 @@
-# -*- coding: utf-8 -*-
-# vim: ft=yaml
----
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This config file is used to test a multi-node deployment using a local
-# dispatcher. This setup is not recommended for production use.
-
-# The variables commented out are the default values that the formula uses.
-# The uncommented values are REQUIRED values. If you don't set them, running
-# this formula will fail.
-arvados:
-  ### GENERAL CONFIG
-  version: '__VERSION__'
-  ## It makes little sense to disable this flag, but you can, if you want :)
-  # use_upstream_repo: true
-
-  ## Repo URL is built with grains values. If desired, it can be completely
-  ## overwritten with the pillar parameter 'repo_url'
-  # repo:
-  #   humanname: Arvados Official Repository
-
-  release: __RELEASE__
-
-  ## IMPORTANT!!!!!
-  ## api, workbench and shell require some gems, so you need to make sure ruby
-  ## and deps are installed in order to install and compile the gems.
-  ## We default to `false` in these two variables as it's expected you already
-  ## manage OS packages with some other tool and you don't want us messing up
-  ## with your setup.
-  ruby:
-    ## We set these to `true` here for testing purposes.
-    ## They both default to `false`.
-    manage_ruby: true
-    manage_gems_deps: true
-    # pkg: ruby
-    # gems_deps:
-    #     - curl
-    #     - g++
-    #     - gcc
-    #     - git
-    #     - libcurl4
-    #     - libcurl4-gnutls-dev
-    #     - libpq-dev
-    #     - libxml2
-    #     - libxml2-dev
-    #     - make
-    #     - python3-dev
-    #     - ruby-dev
-    #     - zlib1g-dev
-
-  # config:
-  #   file: /etc/arvados/config.yml
-  #   user: root
-  ## IMPORTANT!!!!!
-  ## If you're intalling any of the rails apps (api, workbench), the group
-  ## should be set to that of the web server, usually `www-data`
-  #   group: root
-  #   mode: 640
-
-  ### ARVADOS CLUSTER CONFIG
-  cluster:
-    name: __CLUSTER__
-    domain: __DOMAIN__
-
-    database:
-      # max concurrent connections per arvados server daemon
-      # connection_pool_max: 32
-      name: __CLUSTER___arvados
-      host: 127.0.0.1
-      password: "__DATABASE_PASSWORD__"
-      user: __CLUSTER___arvados
-      extra_conn_params:
-        client_encoding: UTF8
-      # Centos7 does not enable SSL by default, so we disable
-      # it here just for testing of the formula purposes only.
-      # You should not do this in production, and should
-      # configure Postgres certificates correctly
-      {%- if grains.os_family in ('RedHat',) %}
-        sslmode: disable
-      {%- endif %}
-
-    tls:
-      # certificate: ''
-      # key: ''
-      # When using arvados-snakeoil certs set insecure: true
-      insecure: true
-
-    resources:
-      virtual_machines:
-        shell:
-          name: shell
-          backend: __SHELL_INT_IP__
-          port: 4200
-
-    ### TOKENS
-    tokens:
-      system_root: __SYSTEM_ROOT_TOKEN__
-      management: __MANAGEMENT_TOKEN__
-      anonymous_user: __ANONYMOUS_USER_TOKEN__
-
-    ### KEYS
-    secrets:
-      blob_signing_key: __BLOB_SIGNING_KEY__
-      workbench_secret_key: __WORKBENCH_SECRET_KEY__
-
-    Login:
-      Test:
-        Enable: true
-        Users:
-          __INITIAL_USER__:
-            Email: __INITIAL_USER_EMAIL__
-            Password: __INITIAL_USER_PASSWORD__
-
-    ### VOLUMES
-    ## This should usually match all your `keepstore` instances
-    Volumes:
-      # the volume name will be composed with
-      # <cluster>-nyw5e-<volume>
-      __CLUSTER__-nyw5e-000000000000000:
-        AccessViaHosts:
-          'http://__KEEPSTORE0_INT_IP__:25107':
-            ReadOnly: false
-        Replication: 2
-        Driver: Directory
-        DriverParameters:
-          Root: /tmp
-      __CLUSTER__-nyw5e-000000000000001:
-        AccessViaHosts:
-          'http://__KEEPSTORE1_INT_IP__:25107':
-            ReadOnly: false
-        Replication: 2
-        Driver: Directory
-        DriverParameters:
-          Root: /tmp
-
-    Containers:
-      LocalKeepBlobBuffersPerVCPU: 0
-
-    Users:
-      NewUsersAreActive: true
-      AutoAdminFirstUser: true
-      AutoSetupNewUsers: true
-      AutoSetupNewUsersWithRepository: true
-
-    Services:
-      Controller:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__'
-        InternalURLs:
-          'http://localhost:8003': {}
-      Keepbalance:
-        InternalURLs:
-          'http://__CONTROLLER_INT_IP__:9005': {}
-      Keepproxy:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__KEEP_EXT_SSL_PORT__'
-        InternalURLs:
-          'http://__KEEP_INT_IP__:25100': {}
-      Keepstore:
-        InternalURLs:
-          'http://__KEEPSTORE0_INT_IP__:25107': {}
-          'http://__KEEPSTORE1_INT_IP__:25107': {}
-      RailsAPI:
-        InternalURLs:
-          'http://localhost:8004': {}
-      WebDAV:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__'
-        InternalURLs:
-          'http://localhost:9002': {}
-      WebDAVDownload:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__KEEPWEB_EXT_SSL_PORT__'
-      WebShell:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__WEBSHELL_EXT_SSL_PORT__'
-      Websocket:
-        ExternalURL: 'wss://__CLUSTER__.__DOMAIN__:__WEBSOCKET_EXT_SSL_PORT__/websocket'
-        InternalURLs:
-          'http://__WEBSOCKET_INT_IP__:8005': {}
-      Workbench1:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__WORKBENCH1_EXT_SSL_PORT__'
-      Workbench2:
-        ExternalURL: 'https://__CLUSTER__.__DOMAIN__:__WORKBENCH2_EXT_SSL_PORT__'
index 6d561a22b991947e25d1603c7d17314fbe19506d..e5ff7be4e7a95f621932ab542a675afd253aa6ce 100755 (executable)
@@ -102,7 +102,7 @@ loadconfig() {
        echo "Must be run from initialized setup dir, maybe you need to 'initialize' first?"
     fi
     source ${CONFIG_FILE}
-    GITTARGET=.arvados-deploy-config-${CLUSTER}
+    GITTARGET=arvados-deploy-config-${CLUSTER}
 }
 
 subcmd="$1"
index 638e5de800169bf9c8f3ea5641d61cbdff2e68f1..173ed15b6881496abaffdba78fbf2c7991513434 100755 (executable)
 
 set -o pipefail
 
-_exit_handler() {
-  local rc="$?"
-  trap - EXIT
-  if [ "$rc" -ne 0 ]; then
-    echo "Error occurred ($rc) while running $0 at line $1 : $BASH_COMMAND"
-  fi
-  exit "$rc"
-}
-
-trap '_exit_handler $LINENO' EXIT ERR
-
 # capture the directory that the script is running from
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
@@ -366,9 +355,9 @@ echo "...arvados"
 git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
 
 # If we want to try a specific branch of the formula
-if [ "x${BRANCH}" != "x" -a $(git rev-parse --abbrev-ref HEAD) != "${BRANCH}" ]; then
+if [ "x${BRANCH}" != "x" ]; then
   ( cd ${F_DIR}/arvados && git checkout --quiet -t origin/"${BRANCH}" -b "${BRANCH}" )
-elif [ "x${ARVADOS_TAG}" != "x" -a $(git rev-parse --abbrev-ref HEAD) != "${ARVADOS_TAG}" ]; then
+elif [ "x${ARVADOS_TAG}" != "x" ]; then
 ( cd ${F_DIR}/arvados && git checkout --quiet tags/"${ARVADOS_TAG}" -b "${ARVADOS_TAG}" )
 fi
 
@@ -706,7 +695,6 @@ else
         sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
       ;;
       "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
-        NGINX_INSTALL_SOURCE="install_from_repo"
         # States
         if [ "${R}" = "workbench" ]; then
           NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
@@ -787,7 +775,7 @@ else
                     s#__CERT_PEM__#/etc/nginx/ssl/arvados-${R}.pem#g;
                     s#__CERT_KEY__#/etc/nginx/ssl/arvados-${R}.key#g" \
             ${P_DIR}/nginx_${R}_configuration.sls
-            grep -q ${R}$ ${P_DIR}/extra_custom_certs.sls || echo "  - ${R}" >> ${P_DIR}/extra_custom_certs.sls
+            grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo "  - ${R}" >> ${P_DIR}/extra_custom_certs.sls
           fi
         fi
         # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
@@ -832,17 +820,15 @@ fi
 
 # Leave a copy of the Arvados CA so the user can copy it where it's required
 if [ "$DEV_MODE" = "yes" ]; then
-  ARVADOS_SNAKEOIL_CA_DEST_FILE="${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem"
-
+  echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
   # If running in a vagrant VM, also add default user to docker group
   if [ "x${VAGRANT}" = "xyes" ]; then
+    cp /etc/ssl/certs/arvados-snakeoil-ca.pem /vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
+
     echo "Adding the vagrant user to the docker group"
     usermod -a -G docker vagrant
-    ARVADOS_SNAKEOIL_CA_DEST_FILE="/vagrant/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem"
-  fi
-  if [ -f /etc/ssl/certs/arvados-snakeoil-ca.pem ]; then
-    echo "Copying the Arvados CA certificate to the installer dir, so you can import it"
-    cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${ARVADOS_SNAKEOIL_CA_DEST_FILE}
+  else
+    cp /etc/ssl/certs/arvados-snakeoil-ca.pem ${SCRIPT_DIR}/${CLUSTER}.${DOMAIN}-arvados-snakeoil-ca.pem
   fi
 fi
 
index 42ab716642739dee5d0d9a76b51f92d24146be7c..cf43273a14d584b390079400b096f12ec1e2d683 100755 (executable)
@@ -9,6 +9,14 @@ export ARVADOS_API_HOST_INSECURE=true
 
 set -o pipefail
 
+# First, validate that the CA is installed and that we can query it with no errors.
+if ! curl -s -o /dev/null https://${ARVADOS_API_HOST}/users/welcome?return_to=%2F; then
+  echo "The Arvados CA was not correctly installed. Although some components will work,"
+  echo "others won't. Please verify that the CA cert file was installed correctly and"
+  echo "retry running these tests."
+  exit 1
+fi
+
 # https://doc.arvados.org/v2.0/install/install-jobs-image.html
 echo "Creating Arvados Standard Docker Images project"
 uuid_prefix=$(arv --format=uuid user current | cut -d- -f1)