docs(single_host): fix hostnames and tests
authorJavier Bértoli <javier@netmanagers.com.ar>
Mon, 23 Nov 2020 13:50:09 +0000 (10:50 -0300)
committerJavier Bértoli <javier@netmanagers.com.ar>
Tue, 24 Nov 2020 11:12:17 +0000 (08:12 -0300)
25 files changed:
arvados/api/package/clean.sls
arvados/api/package/install.sls
arvados/dispatcher/service/files/default/crunch-dispatch-local-credentials.tmpl
arvados/repo/clean.sls
arvados/repo/install.sls
arvados/ruby/package/clean.sls
arvados/shell/package/clean.sls
arvados/shell/package/install.sls
kitchen.yml
pillar.example
test/integration/api/controls/config_spec.rb
test/integration/keepweb/controls/config_spec.rb
test/integration/websocket/controls/config_spec.rb
test/integration/workbench/controls/config_spec.rb
test/salt/pillar/arvados.sls
test/salt/pillar/arvados_dev.sls
test/salt/pillar/examples/nginx_api_configuration.sls
test/salt/pillar/examples/nginx_controller_configuration.sls
test/salt/pillar/examples/nginx_keepproxy_configuration.sls
test/salt/pillar/examples/nginx_keepweb_configuration.sls
test/salt/pillar/examples/nginx_webshell_configuration.sls
test/salt/pillar/examples/nginx_websocket_configuration.sls
test/salt/pillar/examples/nginx_workbench_configuration.sls
test/salt/states/example_add_snakeoil_certs/init.sls
test/salt/states/example_single_host_host_entries/init.sls

index 74d807e6ae502b84cfd80a5972cc9e52003129fa..52e3650612d571248e703495bd42b92a6cd8ff55 100644 (file)
@@ -5,13 +5,13 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% for gm in arvados.api.gem.name %}
+{%- for gm in arvados.api.gem.name %}
 arvados-api-package-clean-gem-{{ gm }}-removed:
   gem.removed:
     - name: {{ gm }}
     - require_in:
       - pkg: arvados-api-package-clean-gems-deps-pkg-removed
-{% endfor %}
+{%- endfor %}
 
 arvados-api-package-clean-gems-deps-pkg-removed:
   pkg.removed:
index c157acd0a50164be6d863cf00e116e3dd9850305..068d4a1afd4d14fcc0025bb9d8b6547b3cbed21a 100644 (file)
@@ -21,7 +21,7 @@ arvados-api-package-install-gems-deps-pkg-installed:
     - pkgs: {{ arvados.ruby.gems_deps | unique | json }}
     - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true"
 
-{% for gm in arvados.api.gem.name | unique %}
+{%- for gm in arvados.api.gem.name | unique %}
 arvados-api-package-install-gem-{{ gm }}-installed:
   gem.installed:
     - name: {{ gm }}
@@ -32,7 +32,7 @@ arvados-api-package-install-gem-{{ gm }}-installed:
       {%- endif %}
     - require_in:
       - pkg: arvados-api-package-install-pkg-installed
-{% endfor %}
+{%- endfor %}
 
 arvados-api-package-install-pkg-installed:
   pkg.installed:
index 2646df875f68fd5452e04a9d9e9475652366a037..a852bc87fcde05ca79f2b14c3ee8f1c5b4240a78 100644 (file)
@@ -2,8 +2,6 @@
 # File managed by Salt at <{{ source }}>.
 # Your changes will be overwritten.
 ########################################################################
-# ARVADOS_API_HOST= arvados.cluster.Services.RailsAPI.InternalURLs:main
-# ARVADOS_API_HOST={% for key in arvados.cluster.Services.Controller.InternalURLs %}{{ key | regex_replace('^http(s?)://', '', ignorecase=true) }}{% endfor %}
 ARVADOS_API_HOST={{ arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) }}
 ARVADOS_API_HOST_INSECURE={{ '1' if arvados.cluster.tls.insecure | default('0') }}
 ARVADOS_API_TOKEN={{ arvados.cluster.tokens.system_root }}
index 55cf601f840fb1152d0cd88f2d1b80e6ed937bfb..3ab71182c229876c6e94cce4a49631b02c82b1f4 100644 (file)
@@ -5,8 +5,8 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% if arvados.use_upstream_repo -%}
-  {% if grains.get('os_family') == 'Debian' -%}
+{%- if arvados.use_upstream_repo %}
+  {%- if grains.get('os_family') == 'Debian' %}
 arvados-repo-clean-repo-absent:
   pkgrepo.absent:
     - file: {{ arvados.repo.file }}
index 57d1ed222ef2a5409f47489124fd180fb9011037..1146f30e5f2d004171923edb84232f9725182a03 100644 (file)
@@ -5,8 +5,8 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{%- if arvados.use_upstream_repo -%}
-  {%- if grains.get('os_family') == 'Debian' -%}
+{%- if arvados.use_upstream_repo %}
+  {%- if grains.get('os_family') == 'Debian' %}
     {%- if arvados.release == 'testing' %}
       {%- set release = grains.get('lsb_distrib_codename') ~ '-testing' %}
     {%- elif arvados.release == 'development' %}
index 85941be14d6ff9d0d2db6c2f3fc1a6019cc1ac2e..cd5f32a02f662a65262d388ee8656c636fba992d 100644 (file)
@@ -5,13 +5,13 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% for gm in arvados.shell.gem.name %}
+{%- for gm in arvados.shell.gem.name %}
 arvados-shell-package-clean-gem-{{ gm }}-removed:
   gem.removed:
     - name: {{ gm }}
     - require_in:
       - pkg: arvados-shell-package-clean-gems-deps-pkg-removed
-{% endfor %}
+{%- endfor %}
 
 arvados-shell-package-clean-gems-deps-pkg-removed:
   pkg.removed:
index 85941be14d6ff9d0d2db6c2f3fc1a6019cc1ac2e..cd5f32a02f662a65262d388ee8656c636fba992d 100644 (file)
@@ -5,13 +5,13 @@
 {%- set tplroot = tpldir.split('/')[0] %}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 
-{% for gm in arvados.shell.gem.name %}
+{%- for gm in arvados.shell.gem.name %}
 arvados-shell-package-clean-gem-{{ gm }}-removed:
   gem.removed:
     - name: {{ gm }}
     - require_in:
       - pkg: arvados-shell-package-clean-gems-deps-pkg-removed
-{% endfor %}
+{%- endfor %}
 
 arvados-shell-package-clean-gems-deps-pkg-removed:
   pkg.removed:
index 8ebfd5916419c95737bdc26971921c239f1b15c5..b1ad75ee68a468d76663337ac84b21e4ce1595c3 100644 (file)
@@ -37,7 +37,7 @@ arvados-shell-package-install-gems-deps-pkg-installed:
     - pkgs: {{ arvados.ruby.gems_deps | json }}
     - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true"
 
-{% for gm in arvados.shell.gem.name %}
+{%- for gm in arvados.shell.gem.name %}
 arvados-shell-package-install-gem-{{ gm }}-installed:
   gem.installed:
     - name: {{ gm }}
@@ -46,4 +46,4 @@ arvados-shell-package-install-gem-{{ gm }}-installed:
       - {{ ruby_dep }}: arvados-ruby-package-install-ruby-{{ ruby_dep }}-installed
       {%- endif %}
       - pkg: arvados-shell-package-install-gems-deps-pkg-installed
-{% endfor %}
+{%- endfor %}
index 0a9c7ab29210798a6983ff5d1953245b32959e4b..4067e1c7f6ec5fc3c1bb6742b52b17917b0408e0 100644 (file)
@@ -104,6 +104,7 @@ suites:
       state_top:
         base:
           '*':
+            - example_single_host_host_entries
             - example_add_snakeoil_certs
             - locale
             - nginx.passenger
@@ -145,6 +146,8 @@ suites:
         example_nginx_controller.sls: test/salt/pillar/examples/nginx_controller_configuration.sls
         # yamllint enable rule:line-length
       dependencies:
+        - name: example_single_host_host_entries
+          path: test/salt/states
         - name: example_add_snakeoil_certs
           path: test/salt/states
         - name: locale
@@ -172,6 +175,7 @@ suites:
       state_top:
         base:
           '*':
+            - example_single_host_host_entries
             - example_add_snakeoil_certs
             - nginx.passenger
             - arvados.repo
@@ -193,6 +197,8 @@ suites:
         example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls
         # yamllint enable rule:line-length
       dependencies:
+        - name: example_single_host_host_entries
+          path: test/salt/states
         - name: example_add_snakeoil_certs
           path: test/salt/states
         - name: nginx
index 3fc4b8bcd2ae71a58bd4014801ae6296cf54767b..dcf7b5efb5e987b163dedf5fdc969a1e35505730 100644 (file)
@@ -82,19 +82,22 @@ arvados:
 
     ### TOKENS
     tokens:
-      system_root: changeme_system_root_token
-      management: changeme_management_token
-      rails_secret: changeme_rails_secret_token
-      anonymous_user: changeme_anonymous_user_token
+      # SystemRootToken has to be alphanumeric, it does not accept underscores
+      # or special characters. See
+      # https://dev.arvados.org/issues/17150
+      system_root: changemesystemroottoken
+      management: changememanagementtoken
+      rails_secret: changemerailssecrettoken
+      anonymous_user: changemeanonymoususertoken
 
     ### KEYS
     secrets:
-      blob_signing_key: changeme_blob_signing_key
-      workbench_secret_key: changeme_workbench_secret_key
-      dispatcher_access_key: changeme_dispatcher_access_key
-      dispatcher_secret_key: changeme_dispatcher_secret_key
-      keep_access_key: changeme_keep_access_key
-      keep_secret_key: changeme_keep_secret_key
+      blob_signing_key: changemeblobsigningkey
+      workbench_secret_key: changemeworkbenchsecretkey
+      dispatcher_access_key: changemedispatcheraccesskey
+      dispatcher_secret_key: changemedispatchersecretkey
+      keep_access_key: changemekeepaccesskey
+      keep_secret_key: changemekeepsecretkey
 
     AuditLogs:
       Section_to_ignore:
index 5d8afc4158db3eedea665c594a4077b34097e3c6..4585b95b8c829144f8ac1a1cfb117ff5503434b6 100644 (file)
@@ -2,13 +2,13 @@
 
 api_stanza = <<-API_STANZA
     API:
-      RailsSessionSecretToken: "changeme_rails_secret_token"
+      RailsSessionSecretToken: "changemerailssecrettoken"
 API_STANZA
 
 rails_stanza = <<-RAILS_STANZA
       RailsAPI:
         InternalURLs:
-          http://127.0.0.2:8004: {}
+          http://api.internal:8004: {}
 RAILS_STANZA
 
 group = case os[:name]
index 6ac0db84ed5abe48af4d01dd493563b7d370205a..3a746366b826388cb4f691a2b45517ba8a984298 100644 (file)
@@ -4,7 +4,7 @@ keepweb_stanza = <<-KEEPWEB_STANZA
       WebDAV:
         ExternalURL: https://collections.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:9002: {}
+          http://collections.internal:9002: {}
       WebDAVDownload:
         ExternalURL: https://download.fixme.example.net
 KEEPWEB_STANZA
index 0152476de14d594f2ef875a6243ca96d3ea93186..d5289c9c1788c5a87c8b8f3dd2f0f25a00728d25 100644 (file)
@@ -4,7 +4,7 @@ websocket_stanza = <<-WEBSOCKET_STANZA
       Websocket:
         ExternalURL: wss://ws.fixme.example.net/websocket
         InternalURLs:
-          http://127.0.0.2:8005: {}
+          http://ws.internal:8005: {}
 WEBSOCKET_STANZA
 
 group = case os[:name]
index f8be4d469870425ba9cbf97bbbccf0b8f96c3bea..9a14383d7ed27d0210a804993e7ecb316cdf8b97 100644 (file)
@@ -2,7 +2,7 @@
 
 workbench_config = <<-WORKBENCH_STANZA
     Workbench:
-      SecretKeyBase: "changeme_workbench_secret_key"
+      SecretKeyBase: "changemeworkbenchsecretkey"
       SiteName: FIXME
 WORKBENCH_STANZA
 
index f81d2ec72d2ce53d3d28ddba2f6485221b89fdcb..9e1ccbc0b220b7a9bcc890f27821509b092ace91 100644 (file)
@@ -100,7 +100,7 @@ arvados:
       Controller:
         ExternalURL: https://fixme.example.net
         InternalURLs:
-          http://127.0.0.2:8003: {}
+          http://controller.internal:8003: {}
       DispatchCloud:
         InternalURLs:
           http://fixme.example.net:9006: {}
@@ -110,17 +110,17 @@ arvados:
       Keepproxy:
         ExternalURL: https://keep.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:25100: {}
+          http://keep.internal:25100: {}
       Keepstore:
         InternalURLs:
           http://keep0.fixme.example.net:25107: {}
       RailsAPI:
         InternalURLs:
-          http://127.0.0.2:8004: {}
+          http://api.internal:8004: {}
       WebDAV:
         ExternalURL: https://collections.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:9002: {}
+          http://collections.internal:9002: {}
       WebDAVDownload:
         ExternalURL: https://download.fixme.example.net
       WebShell:
@@ -128,7 +128,7 @@ arvados:
       Websocket:
         ExternalURL: wss://ws.fixme.example.net/websocket
         InternalURLs:
-          http://127.0.0.2:8005: {}
+          http://ws.internal:8005: {}
       Workbench1:
         ExternalURL: https://workbench.fixme.example.net
       Workbench2:
index 1faae6c519e3f7fdd5ad7f7b274ca7459d38566b..51d650e027c8045e7494f31259577be98f335b4c 100644 (file)
@@ -118,7 +118,7 @@ arvados:
       Controller:
         ExternalURL: https://fixme.example.net
         InternalURLs:
-          http://127.0.0.2:8003: {}
+          http://controller.internal:8003: {}
       DispatchCloud:
         InternalURLs:
           http://fixme.example.net:9006: {}
@@ -128,17 +128,17 @@ arvados:
       Keepproxy:
         ExternalURL: https://keep.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:25100: {}
+          http://keep.internal:25100: {}
       Keepstore:
         InternalURLs:
           http://keep0.fixme.example.net:25107: {}
       RailsAPI:
         InternalURLs:
-          http://127.0.0.2:8004: {}
+          http://api.internal:8004: {}
       WebDAV:
         ExternalURL: https://collections.fixme.example.net
         InternalURLs:
-          http://127.0.0.2:9002: {}
+          http://collections.internal:9002: {}
       WebDAVDownload:
         ExternalURL: https://download.fixme.example.net
       WebShell:
@@ -146,7 +146,7 @@ arvados:
       Websocket:
         ExternalURL: wss://ws.fixme.example.net/websocket
         InternalURLs:
-          http://127.0.0.2:8005: {}
+          http://ws.internal:8005: {}
       Workbench1:
         ExternalURL: https://workbench.fixme.example.net
       Workbench2:
index 3313eab0b41ea6eed0bfca04e400918e0923123b..e64ed11be953a58083bc04a9e28a952b6f608908 100644 (file)
@@ -20,7 +20,7 @@ nginx:
         overwrite: true
         config:
           - server:
-            - listen: '127.0.0.2:8004'
+            - listen: 'api.internal:8004'
             - server_name: api
             - root: /var/www/arvados-api/current/public
             - index:  index.html index.htm
index 3e95cce58649daa8c943c0c7670b5b6807d34b5a..a79bd41e9a735e5c315f4ee66ad859b338801449 100644 (file)
@@ -10,7 +10,7 @@ nginx:
           default: 1
           '127.0.0.0/8': 0
         upstream controller_upstream:
-          - server: '127.0.0.2:8003  fail_timeout=10s'
+          - server: 'controller.internal:8003  fail_timeout=10s'
 
   ### SITES
   servers:
index c20d7bc04dcfa173017d560075f596ab2ee10be9..f1729e7172b248c726be4211df96599f2db3d01d 100644 (file)
@@ -7,7 +7,7 @@ nginx:
       ### STREAMS
       http:
         upstream keepproxy_upstream:
-          - server: '127.0.0.2:25100 fail_timeout=10s'
+          - server: 'keep.internal:25100 fail_timeout=10s'
 
   servers:
     managed:
index dfb02f64230292dfc7512bc28aa02d2e0110eb49..1250d89d2700895ca35837cda677485c0ee670e4 100644 (file)
@@ -7,7 +7,7 @@ nginx:
       ### STREAMS
       http:
         upstream collections_downloads_upstream:
-          - server: '127.0.0.2:9002 fail_timeout=10s'
+          - server: 'collections.internal:9002 fail_timeout=10s'
 
   servers:
     managed:
index c1a8907d2239bfe00c2d21a7328c8ddc0b9faaef..022cd36ac5635acb7ddad417ce7b90a3c1707cf9 100644 (file)
@@ -8,7 +8,7 @@ nginx:
       ### STREAMS
       http:
         upstream webshell_upstream:
-          - server: '127.0.0.2:4200 fail_timeout=10s'
+          - server: 'shell.internal:4200 fail_timeout=10s'
 
   ### SITES
   servers:
index 48e9db26c8e8d496088306d180ced0f4410ea708..f50f40ea1988261d050982b769cbb1b2e11b1fa9 100644 (file)
@@ -7,7 +7,7 @@ nginx:
       ### STREAMS
       http:
         upstream websocket_upstream:
-          - server: '127.0.0.2:8005 fail_timeout=10s'
+          - server: 'ws.internal:8005 fail_timeout=10s'
 
   servers:
     managed:
index 5b8e0e2fee9b3a271ff8c9a62448551bb14a1e7e..fbadc58259f2ec8ca810a0b7673b122329aad491 100644 (file)
@@ -19,7 +19,7 @@ nginx:
       ### STREAMS
       http:
         upstream workbench_upstream:
-          - server: '127.0.0.2:9000 fail_timeout=10s'
+          - server: 'workbench.internal:9000 fail_timeout=10s'
 
   ### SITES
   servers:
@@ -67,7 +67,7 @@ nginx:
         overwrite: true
         config:
           - server:
-            - listen: '127.0.0.2:9000'
+            - listen: 'workbench.internal:9000'
             - server_name: workbench
             - root: /var/www/arvados-workbench/current/public
             - index:  index.html index.htm
index 278ccd08a28dec0f968e7764667b1c41ee11612c..158abcc7783244e572bc1df12f61fdecc38b8cbf 100644 (file)
@@ -1,7 +1,7 @@
-{% set curr_tpldir = tpldir %}
-{% set tpldir = 'arvados' %}
-{% from "arvados/map.jinja" import arvados with context %}
-{% set tpldir = curr_tpldir %}
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- set tpldir = curr_tpldir %}
 
 snake_oil_certs:
   pkg.installed:
@@ -15,7 +15,6 @@ snake_oil_certs:
         default_md = sha256
         x509_extensions = v3_req
         distinguished_name = dn
-        
         [dn]
         C   = CC
         ST  = SomeState
@@ -24,13 +23,11 @@ snake_oil_certs:
         OU  = R&D
         CN  = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
         emailAddress = admin@{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
-        
         [v3_req]
         subjectAltName = @alt_names
-        
         [alt_names]
         {%- for entry in grains.get('ipv4') %}
-        IP.{{ loop.index }} = {{entry }}
+        IP.{{ loop.index }} = {{ entry }}
         {%- endfor %}
         {%- for entry in [
             'keep',
@@ -49,7 +46,24 @@ snake_oil_certs:
         mkdir -p /etc/ssl/certs/  /etc/ssl/private/ && \
         openssl req -config /tmp/openssl.cnf -new -x509 -days 3650 -nodes -sha256 \
           -out /etc/ssl/certs/ssl-cert-snakeoil.pem \
-          -keyout /etc/ssl/private/ssl-cert-snakeoil.key > /tmp/snake_oil_certs.output 2>&1
+          -keyout /etc/ssl/private/ssl-cert-snakeoil.key > /tmp/snake_oil_certs.output 2>&1 && \
+        chmod 0644 /etc/ssl/certs/ssl-cert-snakeoil.pem && \
+        chmod 0640 /etc/ssl/private/ssl-cert-snakeoil.key
     - unless: test -f /etc/ssl/private/ssl-cert-snakeoil.key
     - require:
       - pkg: openssl
+
+{%- if grains.get('os_family') == 'Debian' %}
+ssl_certs:
+  pkg.installed:
+    - name: ssl-cert
+    - require_in:
+      - sls: postgres
+
+snake_oil_certs_permissions:
+  cmd.run:
+    - name: |
+        chown root:ssl-cert /etc/ssl/private/ssl-cert-snakeoil.key
+    - require:
+      - pkg: ssl_certs
+{%- endif %}
index a043333d106e5e5775089f5249a853732935704a..64254480a9efeee21efff42dc500d095d9ab335c 100644 (file)
@@ -1,7 +1,7 @@
-{% set curr_tpldir = tpldir %}
-{% set tpldir = 'arvados' %}
-{% from "arvados/map.jinja" import arvados with context %}
-{% set tpldir = curr_tpldir %}
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- set tpldir = curr_tpldir %}
 
 arvados_hosts_entries:
   host.present:
@@ -11,15 +11,18 @@ arvados_hosts_entries:
       # FIXME! This just works for our testings.
       # Won't work if the cluster name != host name
       {%- for entry in [
-          'keep',
-          'keep0',
+          'api',
           'collections',
+          'controller',
           'download',
-          'ws',
+          'keep',
+          'keep0',
+          'shell',
           'workbench',
-          'workbench2'
+          'workbench2',
+          'ws',
         ]
       %}
-      - {{ entry }}
+      - {{ entry }}.internal
       - {{ entry }}.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
       {%- endfor %}