From 6c52de7c70c90784df58e6dbc6c43a71b9cc7e7c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Javier=20B=C3=A9rtoli?= Date: Mon, 23 Nov 2020 10:50:09 -0300 Subject: [PATCH] docs(single_host): fix hostnames and tests --- arvados/api/package/clean.sls | 4 +-- arvados/api/package/install.sls | 4 +-- .../crunch-dispatch-local-credentials.tmpl | 2 -- arvados/repo/clean.sls | 4 +-- arvados/repo/install.sls | 4 +-- arvados/ruby/package/clean.sls | 4 +-- arvados/shell/package/clean.sls | 4 +-- arvados/shell/package/install.sls | 4 +-- kitchen.yml | 6 ++++ pillar.example | 23 +++++++------ test/integration/api/controls/config_spec.rb | 4 +-- .../keepweb/controls/config_spec.rb | 2 +- .../websocket/controls/config_spec.rb | 2 +- .../workbench/controls/config_spec.rb | 2 +- test/salt/pillar/arvados.sls | 10 +++--- test/salt/pillar/arvados_dev.sls | 10 +++--- .../examples/nginx_api_configuration.sls | 2 +- .../nginx_controller_configuration.sls | 2 +- .../nginx_keepproxy_configuration.sls | 2 +- .../examples/nginx_keepweb_configuration.sls | 2 +- .../examples/nginx_webshell_configuration.sls | 2 +- .../nginx_websocket_configuration.sls | 2 +- .../nginx_workbench_configuration.sls | 4 +-- .../example_add_snakeoil_certs/init.sls | 32 +++++++++++++------ .../example_single_host_host_entries/init.sls | 21 ++++++------ 25 files changed, 91 insertions(+), 67 deletions(-) diff --git a/arvados/api/package/clean.sls b/arvados/api/package/clean.sls index 74d807e..52e3650 100644 --- a/arvados/api/package/clean.sls +++ b/arvados/api/package/clean.sls @@ -5,13 +5,13 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -{% for gm in arvados.api.gem.name %} +{%- for gm in arvados.api.gem.name %} arvados-api-package-clean-gem-{{ gm }}-removed: gem.removed: - name: {{ gm }} - require_in: - pkg: arvados-api-package-clean-gems-deps-pkg-removed -{% endfor %} +{%- endfor %} arvados-api-package-clean-gems-deps-pkg-removed: pkg.removed: diff --git a/arvados/api/package/install.sls b/arvados/api/package/install.sls index c157acd..068d4a1 100644 --- a/arvados/api/package/install.sls +++ b/arvados/api/package/install.sls @@ -21,7 +21,7 @@ arvados-api-package-install-gems-deps-pkg-installed: - pkgs: {{ arvados.ruby.gems_deps | unique | json }} - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" -{% for gm in arvados.api.gem.name | unique %} +{%- for gm in arvados.api.gem.name | unique %} arvados-api-package-install-gem-{{ gm }}-installed: gem.installed: - name: {{ gm }} @@ -32,7 +32,7 @@ arvados-api-package-install-gem-{{ gm }}-installed: {%- endif %} - require_in: - pkg: arvados-api-package-install-pkg-installed -{% endfor %} +{%- endfor %} arvados-api-package-install-pkg-installed: pkg.installed: diff --git a/arvados/dispatcher/service/files/default/crunch-dispatch-local-credentials.tmpl b/arvados/dispatcher/service/files/default/crunch-dispatch-local-credentials.tmpl index 2646df8..a852bc8 100644 --- a/arvados/dispatcher/service/files/default/crunch-dispatch-local-credentials.tmpl +++ b/arvados/dispatcher/service/files/default/crunch-dispatch-local-credentials.tmpl @@ -2,8 +2,6 @@ # File managed by Salt at <{{ source }}>. # Your changes will be overwritten. ######################################################################## -# ARVADOS_API_HOST= arvados.cluster.Services.RailsAPI.InternalURLs:main -# ARVADOS_API_HOST={% for key in arvados.cluster.Services.Controller.InternalURLs %}{{ key | regex_replace('^http(s?)://', '', ignorecase=true) }}{% endfor %} ARVADOS_API_HOST={{ arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) }} ARVADOS_API_HOST_INSECURE={{ '1' if arvados.cluster.tls.insecure | default('0') }} ARVADOS_API_TOKEN={{ arvados.cluster.tokens.system_root }} diff --git a/arvados/repo/clean.sls b/arvados/repo/clean.sls index 55cf601..3ab7118 100644 --- a/arvados/repo/clean.sls +++ b/arvados/repo/clean.sls @@ -5,8 +5,8 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -{% if arvados.use_upstream_repo -%} - {% if grains.get('os_family') == 'Debian' -%} +{%- if arvados.use_upstream_repo %} + {%- if grains.get('os_family') == 'Debian' %} arvados-repo-clean-repo-absent: pkgrepo.absent: - file: {{ arvados.repo.file }} diff --git a/arvados/repo/install.sls b/arvados/repo/install.sls index 57d1ed2..1146f30 100644 --- a/arvados/repo/install.sls +++ b/arvados/repo/install.sls @@ -5,8 +5,8 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -{%- if arvados.use_upstream_repo -%} - {%- if grains.get('os_family') == 'Debian' -%} +{%- if arvados.use_upstream_repo %} + {%- if grains.get('os_family') == 'Debian' %} {%- if arvados.release == 'testing' %} {%- set release = grains.get('lsb_distrib_codename') ~ '-testing' %} {%- elif arvados.release == 'development' %} diff --git a/arvados/ruby/package/clean.sls b/arvados/ruby/package/clean.sls index 85941be..cd5f32a 100644 --- a/arvados/ruby/package/clean.sls +++ b/arvados/ruby/package/clean.sls @@ -5,13 +5,13 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -{% for gm in arvados.shell.gem.name %} +{%- for gm in arvados.shell.gem.name %} arvados-shell-package-clean-gem-{{ gm }}-removed: gem.removed: - name: {{ gm }} - require_in: - pkg: arvados-shell-package-clean-gems-deps-pkg-removed -{% endfor %} +{%- endfor %} arvados-shell-package-clean-gems-deps-pkg-removed: pkg.removed: diff --git a/arvados/shell/package/clean.sls b/arvados/shell/package/clean.sls index 85941be..cd5f32a 100644 --- a/arvados/shell/package/clean.sls +++ b/arvados/shell/package/clean.sls @@ -5,13 +5,13 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -{% for gm in arvados.shell.gem.name %} +{%- for gm in arvados.shell.gem.name %} arvados-shell-package-clean-gem-{{ gm }}-removed: gem.removed: - name: {{ gm }} - require_in: - pkg: arvados-shell-package-clean-gems-deps-pkg-removed -{% endfor %} +{%- endfor %} arvados-shell-package-clean-gems-deps-pkg-removed: pkg.removed: diff --git a/arvados/shell/package/install.sls b/arvados/shell/package/install.sls index 8ebfd59..b1ad75e 100644 --- a/arvados/shell/package/install.sls +++ b/arvados/shell/package/install.sls @@ -37,7 +37,7 @@ arvados-shell-package-install-gems-deps-pkg-installed: - pkgs: {{ arvados.ruby.gems_deps | json }} - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" -{% for gm in arvados.shell.gem.name %} +{%- for gm in arvados.shell.gem.name %} arvados-shell-package-install-gem-{{ gm }}-installed: gem.installed: - name: {{ gm }} @@ -46,4 +46,4 @@ arvados-shell-package-install-gem-{{ gm }}-installed: - {{ ruby_dep }}: arvados-ruby-package-install-ruby-{{ ruby_dep }}-installed {%- endif %} - pkg: arvados-shell-package-install-gems-deps-pkg-installed -{% endfor %} +{%- endfor %} diff --git a/kitchen.yml b/kitchen.yml index 0a9c7ab..4067e1c 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -104,6 +104,7 @@ suites: state_top: base: '*': + - example_single_host_host_entries - example_add_snakeoil_certs - locale - nginx.passenger @@ -145,6 +146,8 @@ suites: example_nginx_controller.sls: test/salt/pillar/examples/nginx_controller_configuration.sls # yamllint enable rule:line-length dependencies: + - name: example_single_host_host_entries + path: test/salt/states - name: example_add_snakeoil_certs path: test/salt/states - name: locale @@ -172,6 +175,7 @@ suites: state_top: base: '*': + - example_single_host_host_entries - example_add_snakeoil_certs - nginx.passenger - arvados.repo @@ -193,6 +197,8 @@ suites: example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls # yamllint enable rule:line-length dependencies: + - name: example_single_host_host_entries + path: test/salt/states - name: example_add_snakeoil_certs path: test/salt/states - name: nginx diff --git a/pillar.example b/pillar.example index 3fc4b8b..dcf7b5e 100644 --- a/pillar.example +++ b/pillar.example @@ -82,19 +82,22 @@ arvados: ### TOKENS tokens: - system_root: changeme_system_root_token - management: changeme_management_token - rails_secret: changeme_rails_secret_token - anonymous_user: changeme_anonymous_user_token + # SystemRootToken has to be alphanumeric, it does not accept underscores + # or special characters. See + # https://dev.arvados.org/issues/17150 + system_root: changemesystemroottoken + management: changememanagementtoken + rails_secret: changemerailssecrettoken + anonymous_user: changemeanonymoususertoken ### KEYS secrets: - blob_signing_key: changeme_blob_signing_key - workbench_secret_key: changeme_workbench_secret_key - dispatcher_access_key: changeme_dispatcher_access_key - dispatcher_secret_key: changeme_dispatcher_secret_key - keep_access_key: changeme_keep_access_key - keep_secret_key: changeme_keep_secret_key + blob_signing_key: changemeblobsigningkey + workbench_secret_key: changemeworkbenchsecretkey + dispatcher_access_key: changemedispatcheraccesskey + dispatcher_secret_key: changemedispatchersecretkey + keep_access_key: changemekeepaccesskey + keep_secret_key: changemekeepsecretkey AuditLogs: Section_to_ignore: diff --git a/test/integration/api/controls/config_spec.rb b/test/integration/api/controls/config_spec.rb index 5d8afc4..4585b95 100644 --- a/test/integration/api/controls/config_spec.rb +++ b/test/integration/api/controls/config_spec.rb @@ -2,13 +2,13 @@ api_stanza = <<-API_STANZA API: - RailsSessionSecretToken: "changeme_rails_secret_token" + RailsSessionSecretToken: "changemerailssecrettoken" API_STANZA rails_stanza = <<-RAILS_STANZA RailsAPI: InternalURLs: - http://127.0.0.2:8004: {} + http://api.internal:8004: {} RAILS_STANZA group = case os[:name] diff --git a/test/integration/keepweb/controls/config_spec.rb b/test/integration/keepweb/controls/config_spec.rb index 6ac0db8..3a74636 100644 --- a/test/integration/keepweb/controls/config_spec.rb +++ b/test/integration/keepweb/controls/config_spec.rb @@ -4,7 +4,7 @@ keepweb_stanza = <<-KEEPWEB_STANZA WebDAV: ExternalURL: https://collections.fixme.example.net InternalURLs: - http://127.0.0.2:9002: {} + http://collections.internal:9002: {} WebDAVDownload: ExternalURL: https://download.fixme.example.net KEEPWEB_STANZA diff --git a/test/integration/websocket/controls/config_spec.rb b/test/integration/websocket/controls/config_spec.rb index 0152476..d5289c9 100644 --- a/test/integration/websocket/controls/config_spec.rb +++ b/test/integration/websocket/controls/config_spec.rb @@ -4,7 +4,7 @@ websocket_stanza = <<-WEBSOCKET_STANZA Websocket: ExternalURL: wss://ws.fixme.example.net/websocket InternalURLs: - http://127.0.0.2:8005: {} + http://ws.internal:8005: {} WEBSOCKET_STANZA group = case os[:name] diff --git a/test/integration/workbench/controls/config_spec.rb b/test/integration/workbench/controls/config_spec.rb index f8be4d4..9a14383 100644 --- a/test/integration/workbench/controls/config_spec.rb +++ b/test/integration/workbench/controls/config_spec.rb @@ -2,7 +2,7 @@ workbench_config = <<-WORKBENCH_STANZA Workbench: - SecretKeyBase: "changeme_workbench_secret_key" + SecretKeyBase: "changemeworkbenchsecretkey" SiteName: FIXME WORKBENCH_STANZA diff --git a/test/salt/pillar/arvados.sls b/test/salt/pillar/arvados.sls index f81d2ec..9e1ccbc 100644 --- a/test/salt/pillar/arvados.sls +++ b/test/salt/pillar/arvados.sls @@ -100,7 +100,7 @@ arvados: Controller: ExternalURL: https://fixme.example.net InternalURLs: - http://127.0.0.2:8003: {} + http://controller.internal:8003: {} DispatchCloud: InternalURLs: http://fixme.example.net:9006: {} @@ -110,17 +110,17 @@ arvados: Keepproxy: ExternalURL: https://keep.fixme.example.net InternalURLs: - http://127.0.0.2:25100: {} + http://keep.internal:25100: {} Keepstore: InternalURLs: http://keep0.fixme.example.net:25107: {} RailsAPI: InternalURLs: - http://127.0.0.2:8004: {} + http://api.internal:8004: {} WebDAV: ExternalURL: https://collections.fixme.example.net InternalURLs: - http://127.0.0.2:9002: {} + http://collections.internal:9002: {} WebDAVDownload: ExternalURL: https://download.fixme.example.net WebShell: @@ -128,7 +128,7 @@ arvados: Websocket: ExternalURL: wss://ws.fixme.example.net/websocket InternalURLs: - http://127.0.0.2:8005: {} + http://ws.internal:8005: {} Workbench1: ExternalURL: https://workbench.fixme.example.net Workbench2: diff --git a/test/salt/pillar/arvados_dev.sls b/test/salt/pillar/arvados_dev.sls index 1faae6c..51d650e 100644 --- a/test/salt/pillar/arvados_dev.sls +++ b/test/salt/pillar/arvados_dev.sls @@ -118,7 +118,7 @@ arvados: Controller: ExternalURL: https://fixme.example.net InternalURLs: - http://127.0.0.2:8003: {} + http://controller.internal:8003: {} DispatchCloud: InternalURLs: http://fixme.example.net:9006: {} @@ -128,17 +128,17 @@ arvados: Keepproxy: ExternalURL: https://keep.fixme.example.net InternalURLs: - http://127.0.0.2:25100: {} + http://keep.internal:25100: {} Keepstore: InternalURLs: http://keep0.fixme.example.net:25107: {} RailsAPI: InternalURLs: - http://127.0.0.2:8004: {} + http://api.internal:8004: {} WebDAV: ExternalURL: https://collections.fixme.example.net InternalURLs: - http://127.0.0.2:9002: {} + http://collections.internal:9002: {} WebDAVDownload: ExternalURL: https://download.fixme.example.net WebShell: @@ -146,7 +146,7 @@ arvados: Websocket: ExternalURL: wss://ws.fixme.example.net/websocket InternalURLs: - http://127.0.0.2:8005: {} + http://ws.internal:8005: {} Workbench1: ExternalURL: https://workbench.fixme.example.net Workbench2: diff --git a/test/salt/pillar/examples/nginx_api_configuration.sls b/test/salt/pillar/examples/nginx_api_configuration.sls index 3313eab..e64ed11 100644 --- a/test/salt/pillar/examples/nginx_api_configuration.sls +++ b/test/salt/pillar/examples/nginx_api_configuration.sls @@ -20,7 +20,7 @@ nginx: overwrite: true config: - server: - - listen: '127.0.0.2:8004' + - listen: 'api.internal:8004' - server_name: api - root: /var/www/arvados-api/current/public - index: index.html index.htm diff --git a/test/salt/pillar/examples/nginx_controller_configuration.sls b/test/salt/pillar/examples/nginx_controller_configuration.sls index 3e95cce..a79bd41 100644 --- a/test/salt/pillar/examples/nginx_controller_configuration.sls +++ b/test/salt/pillar/examples/nginx_controller_configuration.sls @@ -10,7 +10,7 @@ nginx: default: 1 '127.0.0.0/8': 0 upstream controller_upstream: - - server: '127.0.0.2:8003 fail_timeout=10s' + - server: 'controller.internal:8003 fail_timeout=10s' ### SITES servers: diff --git a/test/salt/pillar/examples/nginx_keepproxy_configuration.sls b/test/salt/pillar/examples/nginx_keepproxy_configuration.sls index c20d7bc..f1729e7 100644 --- a/test/salt/pillar/examples/nginx_keepproxy_configuration.sls +++ b/test/salt/pillar/examples/nginx_keepproxy_configuration.sls @@ -7,7 +7,7 @@ nginx: ### STREAMS http: upstream keepproxy_upstream: - - server: '127.0.0.2:25100 fail_timeout=10s' + - server: 'keep.internal:25100 fail_timeout=10s' servers: managed: diff --git a/test/salt/pillar/examples/nginx_keepweb_configuration.sls b/test/salt/pillar/examples/nginx_keepweb_configuration.sls index dfb02f6..1250d89 100644 --- a/test/salt/pillar/examples/nginx_keepweb_configuration.sls +++ b/test/salt/pillar/examples/nginx_keepweb_configuration.sls @@ -7,7 +7,7 @@ nginx: ### STREAMS http: upstream collections_downloads_upstream: - - server: '127.0.0.2:9002 fail_timeout=10s' + - server: 'collections.internal:9002 fail_timeout=10s' servers: managed: diff --git a/test/salt/pillar/examples/nginx_webshell_configuration.sls b/test/salt/pillar/examples/nginx_webshell_configuration.sls index c1a8907..022cd36 100644 --- a/test/salt/pillar/examples/nginx_webshell_configuration.sls +++ b/test/salt/pillar/examples/nginx_webshell_configuration.sls @@ -8,7 +8,7 @@ nginx: ### STREAMS http: upstream webshell_upstream: - - server: '127.0.0.2:4200 fail_timeout=10s' + - server: 'shell.internal:4200 fail_timeout=10s' ### SITES servers: diff --git a/test/salt/pillar/examples/nginx_websocket_configuration.sls b/test/salt/pillar/examples/nginx_websocket_configuration.sls index 48e9db2..f50f40e 100644 --- a/test/salt/pillar/examples/nginx_websocket_configuration.sls +++ b/test/salt/pillar/examples/nginx_websocket_configuration.sls @@ -7,7 +7,7 @@ nginx: ### STREAMS http: upstream websocket_upstream: - - server: '127.0.0.2:8005 fail_timeout=10s' + - server: 'ws.internal:8005 fail_timeout=10s' servers: managed: diff --git a/test/salt/pillar/examples/nginx_workbench_configuration.sls b/test/salt/pillar/examples/nginx_workbench_configuration.sls index 5b8e0e2..fbadc58 100644 --- a/test/salt/pillar/examples/nginx_workbench_configuration.sls +++ b/test/salt/pillar/examples/nginx_workbench_configuration.sls @@ -19,7 +19,7 @@ nginx: ### STREAMS http: upstream workbench_upstream: - - server: '127.0.0.2:9000 fail_timeout=10s' + - server: 'workbench.internal:9000 fail_timeout=10s' ### SITES servers: @@ -67,7 +67,7 @@ nginx: overwrite: true config: - server: - - listen: '127.0.0.2:9000' + - listen: 'workbench.internal:9000' - server_name: workbench - root: /var/www/arvados-workbench/current/public - index: index.html index.htm diff --git a/test/salt/states/example_add_snakeoil_certs/init.sls b/test/salt/states/example_add_snakeoil_certs/init.sls index 278ccd0..158abcc 100644 --- a/test/salt/states/example_add_snakeoil_certs/init.sls +++ b/test/salt/states/example_add_snakeoil_certs/init.sls @@ -1,7 +1,7 @@ -{% set curr_tpldir = tpldir %} -{% set tpldir = 'arvados' %} -{% from "arvados/map.jinja" import arvados with context %} -{% set tpldir = curr_tpldir %} +{%- set curr_tpldir = tpldir %} +{%- set tpldir = 'arvados' %} +{%- from "arvados/map.jinja" import arvados with context %} +{%- set tpldir = curr_tpldir %} snake_oil_certs: pkg.installed: @@ -15,7 +15,6 @@ snake_oil_certs: default_md = sha256 x509_extensions = v3_req distinguished_name = dn - [dn] C = CC ST = SomeState @@ -24,13 +23,11 @@ snake_oil_certs: OU = R&D CN = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }} emailAddress = admin@{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} - [v3_req] subjectAltName = @alt_names - [alt_names] {%- for entry in grains.get('ipv4') %} - IP.{{ loop.index }} = {{entry }} + IP.{{ loop.index }} = {{ entry }} {%- endfor %} {%- for entry in [ 'keep', @@ -49,7 +46,24 @@ snake_oil_certs: mkdir -p /etc/ssl/certs/ /etc/ssl/private/ && \ openssl req -config /tmp/openssl.cnf -new -x509 -days 3650 -nodes -sha256 \ -out /etc/ssl/certs/ssl-cert-snakeoil.pem \ - -keyout /etc/ssl/private/ssl-cert-snakeoil.key > /tmp/snake_oil_certs.output 2>&1 + -keyout /etc/ssl/private/ssl-cert-snakeoil.key > /tmp/snake_oil_certs.output 2>&1 && \ + chmod 0644 /etc/ssl/certs/ssl-cert-snakeoil.pem && \ + chmod 0640 /etc/ssl/private/ssl-cert-snakeoil.key - unless: test -f /etc/ssl/private/ssl-cert-snakeoil.key - require: - pkg: openssl + +{%- if grains.get('os_family') == 'Debian' %} +ssl_certs: + pkg.installed: + - name: ssl-cert + - require_in: + - sls: postgres + +snake_oil_certs_permissions: + cmd.run: + - name: | + chown root:ssl-cert /etc/ssl/private/ssl-cert-snakeoil.key + - require: + - pkg: ssl_certs +{%- endif %} diff --git a/test/salt/states/example_single_host_host_entries/init.sls b/test/salt/states/example_single_host_host_entries/init.sls index a043333..6425448 100644 --- a/test/salt/states/example_single_host_host_entries/init.sls +++ b/test/salt/states/example_single_host_host_entries/init.sls @@ -1,7 +1,7 @@ -{% set curr_tpldir = tpldir %} -{% set tpldir = 'arvados' %} -{% from "arvados/map.jinja" import arvados with context %} -{% set tpldir = curr_tpldir %} +{%- set curr_tpldir = tpldir %} +{%- set tpldir = 'arvados' %} +{%- from "arvados/map.jinja" import arvados with context %} +{%- set tpldir = curr_tpldir %} arvados_hosts_entries: host.present: @@ -11,15 +11,18 @@ arvados_hosts_entries: # FIXME! This just works for our testings. # Won't work if the cluster name != host name {%- for entry in [ - 'keep', - 'keep0', + 'api', 'collections', + 'controller', 'download', - 'ws', + 'keep', + 'keep0', + 'shell', 'workbench', - 'workbench2' + 'workbench2', + 'ws', ] %} - - {{ entry }} + - {{ entry }}.internal - {{ entry }}.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} {%- endfor %} -- 2.30.2