From: Javier Bértoli Date: Fri, 23 Jul 2021 10:26:56 +0000 (-0300) Subject: Merge branch '17750-fix-centos-7-deps' X-Git-Url: https://git.arvados.org/arvados-formula.git/commitdiff_plain/ad8af2094ca06d93dba1f61f3e96587b12e7f667?hp=5c47b29c23a24f020620958e5271b8d8ba5e27e4 Merge branch '17750-fix-centos-7-deps' closes #17750 Arvados-DCO-1.1-Signed-off-by: Javier Bértoli --- diff --git a/arvados/api/package/install.sls b/arvados/api/package/install.sls index 54e20ef..ce0004e 100644 --- a/arvados/api/package/install.sls +++ b/arvados/api/package/install.sls @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- # vim: ft=sls +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} {%- set sls_config_file = tplroot ~ '.config.file' %} @@ -25,6 +29,9 @@ arvados-api-package-install-gems-deps-pkg-installed: arvados-api-package-install-gem-{{ gm }}-installed: gem.installed: - name: {{ gm }} + {%- if arvados.ruby.use_rvm %} + - ruby: {{ arvados.ruby.pkg }} + {%- endif %} - require: - pkg: arvados-api-package-install-gems-deps-pkg-installed {%- if arvados.ruby.manage_ruby %} diff --git a/arvados/controller/package/install.sls b/arvados/controller/package/install.sls index de2166a..62161ea 100644 --- a/arvados/controller/package/install.sls +++ b/arvados/controller/package/install.sls @@ -1,10 +1,43 @@ # -*- coding: utf-8 -*- # vim: ft=sls +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} +{%- set sls_ruby_install = tplroot ~ '.ruby.package.install' %} {%- from tplroot ~ "/map.jinja" import arvados with context %} +{%- if arvados.ruby.manage_ruby %} + {%- set ruby_dep = 'rvm' if arvados.ruby.use_rvm else 'pkg' %} +{%- endif %} + +include: + - {{ sls_ruby_install }} + +arvados-controller-package-install-gems-deps-pkg-installed: + pkg.installed: + - pkgs: {{ arvados.ruby.gems_deps | unique | json }} + - onlyif: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" + +{%- for gm in arvados.api.gem.name | unique %} +arvados-controller-package-install-gem-{{ gm }}-installed: + gem.installed: + - name: {{ gm }} + {%- if arvados.ruby.use_rvm %} + - ruby: {{ arvados.ruby.pkg }} + {%- endif %} + - require: + - pkg: arvados-controller-package-install-gems-deps-pkg-installed + {%- if arvados.ruby.manage_ruby %} + - {{ ruby_dep }}: arvados-ruby-package-install-ruby-{{ ruby_dep }}-installed + {%- endif %} + - require_in: + - pkg: arvados-controller-package-install-pkg-installed +{%- endfor %} + arvados-controller-package-install-pkg-installed: pkg.installed: - name: {{ arvados.controller.pkg.name }} diff --git a/arvados/controller/resources/virtual_machines.sls b/arvados/controller/resources/virtual_machines.sls index 3ee67f3..e0917c0 100644 --- a/arvados/controller/resources/virtual_machines.sls +++ b/arvados/controller/resources/virtual_machines.sls @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- # vim: ft=sls +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} {%- set sls_config_file = tplroot ~ '.config.file' %} @@ -11,6 +15,9 @@ {%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %} {%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %} +{%- set arv_command = '/usr/local/rvm/bin/rvm-exec default arv' + if arvados.ruby.manage_ruby and arvados.ruby.use_rvm + else 'arv' %} include: - ..package - {{ sls_config_file }} @@ -24,8 +31,8 @@ arvados-controller-resources-virtual-machines-jq-pkg-installed: {%- set vm_name = vm_params.name | default(vm) %} {%- set cmd_query_vm_uuid = 'ARVADOS_API_TOKEN=' ~ api_token ~ - ' ARVADOS_API_HOST=' ~ api_host ~ - ' arv --short virtual_machine list' ~ + ' ARVADOS_API_HOST=' ~ api_host ~ ' ' ~ + arv_command ~ ' --short virtual_machine list' ~ ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\'' %} @@ -36,7 +43,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run: - ARVADOS_API_TOKEN: {{ api_token }} - ARVADOS_API_HOST: {{ api_host }} - name: | - arv --format=uuid \ + {{ arv_command }} --format=uuid \ virtual_machine \ create \ --virtual-machine '{"hostname":"{{ vm_name }}" }' @@ -46,6 +53,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run: - require: - pkg: arvados-controller-package-install-pkg-installed - cmd: arvados-controller-service-running-service-ready-cmd-run + - gem: arvados-controller-package-install-gem-arvados-cli-installed # We need to use the UUID generated in the previous command to see if there's a # scoped token for it. There's no easy way to pass the value from a shellout @@ -54,9 +62,13 @@ arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run: # the file content is a token uuid :| arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run: cmd.run: + {%- if arvados.ruby.manage_ruby and arvados.ruby.use_rvm %} + - prepend_path: /usr/local/rvm/gems/{{ arvados.ruby.pkg }}/bin + {%- endif %} - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/{{ vm }} - require: - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run + - gem: arvados-controller-package-install-gem-arvados-cli-installed - unless: - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/{{ vm }} @@ -64,8 +76,8 @@ arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run: # so we need to parse the api_client_authorization list through some jq {%- set cmd_query_scoped_token_url = 'VM_UUID=$(cat /tmp/' ~ vm ~ ') && ' ~ ' ARVADOS_API_TOKEN=' ~ api_token ~ - ' ARVADOS_API_HOST=' ~ api_host ~ - ' arv api_client_authorization list |' ~ + ' ARVADOS_API_HOST=' ~ api_host ~ ' ' ~ + arv_command ~ ' api_client_authorization list |' ~ ' /usr/bin/jq -e \'.items[].scopes[] | select(. == "GET ' ~ '/arvados/v1/virtual_machines/\'${VM_UUID}\'/logins")\' && ' ~ 'unset VM_UUID' @@ -79,7 +91,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run: - ARVADOS_API_HOST: {{ api_host }} - name: | VM_UUID=$(cat /tmp/{{ vm }}) && - arv --format=uuid \ + {{ arv_command }} --format=uuid \ api_client_authorization \ create \ --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/'${VM_UUID}'/logins"]}' @@ -88,5 +100,6 @@ arvados-controller-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run: - pkg: arvados-controller-package-install-pkg-installed - pkg: arvados-controller-resources-virtual-machines-jq-pkg-installed - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run + - gem: arvados-controller-package-install-gem-arvados-cli-installed {%- endfor %} diff --git a/arvados/defaults.yaml b/arvados/defaults.yaml index 001840d..6f44cbb 100644 --- a/arvados/defaults.yaml +++ b/arvados/defaults.yaml @@ -16,6 +16,8 @@ arvados: manage_ruby: false pkg: ruby use_rvm: false + # Only used if using RVM + rvm_gemset: arvados manage_gems_deps: false gems_deps: @@ -65,6 +67,10 @@ arvados: gem: name: - arvados-cli + # Arvados API needs postgresql libs and cli to connect to the database + # and populate it. Please specify the postgresql version that you expect + # to use here + postgresql_version: 12 service: name: nginx port: 8004 @@ -72,6 +78,9 @@ arvados: controller: pkg: name: arvados-controller + gem: + name: + - arvados-cli service: name: arvados-controller port: 8003 @@ -120,6 +129,7 @@ arvados: name: - arvados-client - arvados-src + - libpam-arvados-go - python3-arvados-fuse - python3-arvados-python-client - python3-arvados-cwl-runner diff --git a/arvados/osfamilymap.yaml b/arvados/osfamilymap.yaml index d6e9df1..fae57b6 100644 --- a/arvados/osfamilymap.yaml +++ b/arvados/osfamilymap.yaml @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- # vim: ft=yaml # +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + # Setup variables using grains['os_family'] based logic. # You just need to add the key:values for an `os_family` that differ # from `defaults.yaml` + `osarch.yaml`. @@ -26,7 +30,7 @@ RedHat: manage_ruby: true use_rvm: true - pkg: ruby-2.5.7 + pkg: ruby-2.5.8 gems_deps: - rubygem-bundler - curl @@ -35,12 +39,11 @@ RedHat: - libcurl - libcurl-devel - pam-devel - - postgresql-devel + - centos-release-scl - libxml2 - libxml2-devel - make - python3-devel - - ruby-devel - zlib-devel ##### SHELL shell: diff --git a/arvados/ruby/package/install.sls b/arvados/ruby/package/install.sls index c76c043..8d9ed6b 100644 --- a/arvados/ruby/package/install.sls +++ b/arvados/ruby/package/install.sls @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- # vim: ft=sls +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} @@ -8,6 +12,22 @@ {%- if arvados.ruby.manage_ruby %} {%- if arvados.ruby.use_rvm %} + + # Centos 7 has a too old postgresql package and we need a newer one + {%- if grains.os_family in ('RedHat',) %} +arvados-ruby-package-install-ruby-rvm-deps-centos-scl-release-pkg-installed: + pkg.installed: + - name: centos-release-scl + - require_in: + +arvados-ruby-package-install-ruby-rvm-deps-rh-postgres11-libs-pkg-installed: + pkg.installed: + - name: rh-postgresql{{ arvados.api.postgresql_version }}-postgresql-libs + - unless: rpm -q postgresql{{ arvados.api.postgresql_version }}-libs + - require_in: + - cmd: arvados-ruby-package-install-rvm-cmd-run-curl + {%- endif %} + # Centos 7 has no python3-gnupg package, so using gpg.present # will fail when it can't list the existing keys. # Doing it the hard way diff --git a/arvados/workbench/package/install.sls b/arvados/workbench/package/install.sls index dff47f6..f379f64 100644 --- a/arvados/workbench/package/install.sls +++ b/arvados/workbench/package/install.sls @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- # vim: ft=sls +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} {%- set sls_config_file = tplroot ~ '.config.file' %} @@ -26,4 +30,7 @@ arvados-workbench-package-install-pkg-installed: - name: {{ arvados.workbench.pkg.name }} - version: {{ arvados.version }} - require: + {%- if arvados.ruby.manage_ruby %} + - {{ ruby_dep }}: arvados-ruby-package-install-ruby-{{ ruby_dep }}-installed + {%- endif %} - sls: {{ sls_config_file }} diff --git a/kitchen.yml b/kitchen.yml index 57ec97b..8c1086d 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -12,23 +12,27 @@ driver: privileged: true run_command: /lib/systemd/systemd +provisioner: + name: salt_solo + log_level: debug + salt_install: none + require_chef: false + formula: arvados + salt_copy_filter: + - .kitchen + - .git + platforms: ## SALT `tiamat` - name: debian-10-tiamat-py3 driver: image: saltimages/salt-tiamat-py3:debian-10 - - name: debian-9-tiamat-py3 - driver: - image: saltimages/salt-tiamat-py3:debian-9 - name: ubuntu-2004-tiamat-py3 driver: image: saltimages/salt-tiamat-py3:ubuntu-20.04 - name: ubuntu-1804-tiamat-py3 driver: image: saltimages/salt-tiamat-py3:ubuntu-18.04 - - name: ubuntu-1604-tiamat-py3 - driver: - image: saltimages/salt-tiamat-py3:ubuntu-16.04 - name: centos-7-tiamat-py3 driver: image: saltimages/salt-tiamat-py3:centos-7 @@ -37,55 +41,40 @@ platforms: - name: debian-10-master-py3 driver: image: saltimages/salt-master-py3:debian-10 - - name: debian-9-master-py3 - driver: - image: saltimages/salt-master-py3:debian-9 - name: ubuntu-2004-master-py3 driver: image: saltimages/salt-master-py3:ubuntu-20.04 - name: ubuntu-1804-master-py3 driver: image: saltimages/salt-master-py3:ubuntu-18.04 - - name: ubuntu-1604-master-py3 - driver: - image: saltimages/salt-master-py3:ubuntu-16.04 - name: centos-7-master-py3 driver: image: saltimages/salt-master-py3:centos-7 - ## SALT `3003.0` - - name: debian-10-3003-0-py3 + ## SALT `3003.1` + - name: debian-10-3003-1-py3 driver: - image: saltimages/salt-3003.0-py3:debian-10 - - name: debian-9-3003-0-py3 + image: saltimages/salt-3003.1-py3:debian-10 + - name: ubuntu-2004-3003-1-py3 driver: - image: saltimages/salt-3003.0-py3:debian-9 - - name: ubuntu-2004-3003-0-py3 + image: saltimages/salt-3003.1-py3:ubuntu-20.04 + - name: ubuntu-1804-3003-1-py3 driver: - image: saltimages/salt-3003.0-py3:ubuntu-20.04 - - name: ubuntu-1804-3003-0-py3 + image: saltimages/salt-3003.1-py3:ubuntu-18.04 + - name: centos-7-3003-1-py3 driver: - image: saltimages/salt-3003.0-py3:ubuntu-18.04 - - name: centos-7-3003-0-py3 - driver: - image: saltimages/salt-3003.0-py3:centos-7 + image: saltimages/salt-3003.1-py3:centos-7 ## SALT `3002.6` - name: debian-10-3002-6-py3 driver: image: saltimages/salt-3002.6-py3:debian-10 - - name: debian-9-3002-6-py3 - driver: - image: saltimages/salt-3002.6-py3:debian-9 - name: ubuntu-2004-3002-6-py3 driver: image: saltimages/salt-3002.6-py3:ubuntu-20.04 - name: ubuntu-1804-3002-6-py3 driver: image: saltimages/salt-3002.6-py3:ubuntu-18.04 - - name: ubuntu-1604-3002-6-py3 - driver: - image: saltimages/salt-3002.6-py3:ubuntu-16.04 - name: centos-7-3002-6-py3 driver: image: saltimages/salt-3002.6-py3:centos-7 @@ -94,18 +83,12 @@ platforms: - name: debian-10-3001-7-py3 driver: image: saltimages/salt-3001.7-py3:debian-10 - - name: debian-9-3001-7-py3 - driver: - image: saltimages/salt-3001.7-py3:debian-9 - name: ubuntu-2004-3001-7-py3 driver: image: saltimages/salt-3001.7-py3:ubuntu-20.04 - name: ubuntu-1804-3001-7-py3 driver: image: saltimages/salt-3001.7-py3:ubuntu-18.04 - - name: ubuntu-1604-3001-7-py3 - driver: - image: saltimages/salt-3001.7-py3:ubuntu-16.04 - name: centos-7-3001-7-py3 driver: image: saltimages/salt-3001.7-py3:centos-7 @@ -114,34 +97,15 @@ platforms: - name: debian-10-3000-9-py3 driver: image: saltimages/salt-3000.9-py3:debian-10 - - name: debian-9-3000-9-py3 - driver: - image: saltimages/salt-3000.9-py3:debian-9 - name: ubuntu-1804-3000-9-py3 driver: image: saltimages/salt-3000.9-py3:ubuntu-18.04 - - name: ubuntu-1604-3000-9-py3 - driver: - image: saltimages/salt-3000.9-py3:ubuntu-16.04 - name: centos-7-3000-9-py3 driver: image: saltimages/salt-3000.9-py3:centos-7 - name: ubuntu-1804-3000-9-py2 driver: image: saltimages/salt-3000.9-py2:ubuntu-18.04 - - name: ubuntu-1604-3000-9-py2 - driver: - image: saltimages/salt-3000.9-py2:ubuntu-16.04 - -provisioner: - name: salt_solo - log_level: debug - salt_install: none - require_chef: false - formula: arvados - salt_copy_filter: - - .kitchen - - .git verifier: # https://www.inspec.io/ diff --git a/pillar.example b/pillar.example index ba5f228..78676f1 100644 --- a/pillar.example +++ b/pillar.example @@ -1,6 +1,10 @@ # -*- coding: utf-8 -*- # vim: ft=yaml --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + # The variables commented out are the default values that the formula uses. # The uncommented values are REQUIRED values. If you don't set them, running # this formula will fail. @@ -218,6 +222,9 @@ arvados: # controller: # pkg: # name: arvados-controller +# gem: +# name: +# - arvados-cli # service: # name: arvados-controller # port: 8003 diff --git a/test/integration/api/controls/packages_spec.rb b/test/integration/api/controls/packages_spec.rb index 25641d3..edc2060 100644 --- a/test/integration/api/controls/packages_spec.rb +++ b/test/integration/api/controls/packages_spec.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + control 'arvados api package' do title 'should be installed' @@ -7,3 +11,44 @@ control 'arvados api package' do it { should be_installed } end end + +control 'arvados cli gem' do + title 'should be installed' + + describe gem('arvados-cli') do + it { should be_installed } + end +end + +control 'RVM and dependencies' do + title 'should be installed' + + only_if("Forced requirement for RedHat's family") do + os.redhat? + end + + %w[ + centos-release-scl + curl + gcc + git + libcurl + libcurl-devel + libxml2 + libxml2-devel + make + pam-devel + postgresql12-libs + python3-devel + rubygem-bundler + zlib-devel + ].each do |p| + describe package(p) do + it { should be_installed } + end + end + describe command('/usr/local/rvm/bin/rvm list') do + its(:exit_status) { should eq 0 } + its('stdout') { should match(/ruby-2.5.8/) } + end +end diff --git a/test/salt/pillar/examples/nginx_controller_configuration.sls b/test/salt/pillar/examples/nginx_controller_configuration.sls index a79bd41..787af82 100644 --- a/test/salt/pillar/examples/nginx_controller_configuration.sls +++ b/test/salt/pillar/examples/nginx_controller_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + ### NGINX nginx: ### SERVER @@ -32,6 +36,8 @@ nginx: arvados_controller_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: fixme.example.net @@ -49,8 +55,7 @@ nginx: - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' - proxy_set_header: 'X-External-Client $external_client' - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' - access_log: /var/log/nginx/fixme.example.net.access.log combined - error_log: /var/log/nginx/fixme.example.net.error.log - client_max_body_size: 128m diff --git a/test/salt/pillar/examples/nginx_keepproxy_configuration.sls b/test/salt/pillar/examples/nginx_keepproxy_configuration.sls index f1729e7..d9ed0c6 100644 --- a/test/salt/pillar/examples/nginx_keepproxy_configuration.sls +++ b/test/salt/pillar/examples/nginx_keepproxy_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + ### NGINX nginx: ### SERVER @@ -28,6 +32,8 @@ nginx: arvados_keepproxy_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: keep.fixme.example.net @@ -49,7 +55,6 @@ nginx: - proxy_http_version: '1.1' - proxy_request_buffering: 'off' - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' - access_log: /var/log/nginx/keepproxy.fixme.example.net.access.log combined - error_log: /var/log/nginx/keepproxy.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_keepweb_configuration.sls b/test/salt/pillar/examples/nginx_keepweb_configuration.sls index 1250d89..be18c4d 100644 --- a/test/salt/pillar/examples/nginx_keepweb_configuration.sls +++ b/test/salt/pillar/examples/nginx_keepweb_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + ### NGINX nginx: ### SERVER @@ -29,6 +33,8 @@ nginx: arvados_collections_download_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: collections.fixme.example.net download.fixme.example.net @@ -49,7 +55,6 @@ nginx: - proxy_http_version: '1.1' - proxy_request_buffering: 'off' - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' - access_log: /var/log/nginx/collections.fixme.example.net.access.log combined - error_log: /var/log/nginx/collections.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_passenger.sls b/test/salt/pillar/examples/nginx_passenger.sls index ec79746..7091e28 100644 --- a/test/salt/pillar/examples/nginx_passenger.sls +++ b/test/salt/pillar/examples/nginx_passenger.sls @@ -1,10 +1,16 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {%- if grains.os_family in ('RedHat',) %} {%- set passenger_pkg = 'nginx-mod-http-passenger' %} {%- set passenger_mod = '/usr/lib64/nginx/modules/ngx_http_passenger_module.so' %} + {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.5.8/bin/ruby' %} {%- else %} {%- set passenger_pkg = 'libnginx-mod-http-passenger' %} {%- set passenger_mod = '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %} + {%- set passenger_ruby = '/usr/bin/ruby' %} {%- endif %} ### NGINX @@ -12,6 +18,10 @@ nginx: install_from_phusionpassenger: true lookup: passenger_package: {{ passenger_pkg }} + ### PASSENGER + passenger: + passenger_ruby: {{ passenger_ruby }} + ### SERVER server: config: @@ -52,8 +62,8 @@ nginx: # replace with the IP address of your resolver # - resolver: 127.0.0.1 - ssl_snakeoil.conf: - - ssl_certificate: /etc/ssl/certs/arvados-snakeoil-cert.pem + arvados-snakeoil.conf: + - ssl_certificate: /etc/ssl/private/arvados-snakeoil-cert.pem - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key ### SITES diff --git a/test/salt/pillar/examples/nginx_webshell_configuration.sls b/test/salt/pillar/examples/nginx_webshell_configuration.sls index 050adf9..d228715 100644 --- a/test/salt/pillar/examples/nginx_webshell_configuration.sls +++ b/test/salt/pillar/examples/nginx_webshell_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + # This parameter will be used here to generate a list of upstreams and vhosts. # This dict is here for convenience and should be managed some other way, but the # different ways of orchestration that can be used for this are outside the scope @@ -54,6 +58,8 @@ nginx: arvados_webshell_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: webshell.fixme.example.net @@ -92,8 +98,7 @@ nginx: - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" {%- endfor %} - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' - access_log: /var/log/nginx/webshell.fixme.example.net.access.log combined - error_log: /var/log/nginx/webshell.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_websocket_configuration.sls b/test/salt/pillar/examples/nginx_websocket_configuration.sls index f50f40e..20682bb 100644 --- a/test/salt/pillar/examples/nginx_websocket_configuration.sls +++ b/test/salt/pillar/examples/nginx_websocket_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + ### NGINX nginx: ### SERVER @@ -28,6 +32,8 @@ nginx: arvados_websocket_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: ws.fixme.example.net @@ -50,7 +56,6 @@ nginx: - proxy_http_version: '1.1' - proxy_request_buffering: 'off' - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' - access_log: /var/log/nginx/ws.fixme.example.net.access.log combined - error_log: /var/log/nginx/ws.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_workbench2_configuration.sls b/test/salt/pillar/examples/nginx_workbench2_configuration.sls index c633649..3c3ba4e 100644 --- a/test/salt/pillar/examples/nginx_workbench2_configuration.sls +++ b/test/salt/pillar/examples/nginx_workbench2_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {%- if grains.os_family in ('RedHat',) %} {%- set group = 'nginx' %} {%- else %} @@ -32,6 +36,8 @@ nginx: arvados_workbench2_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: workbench2.fixme.example.net @@ -46,7 +52,6 @@ nginx: - location /config.json: - return: {{ "200 '" ~ '{"API_HOST":"fixme.example.net"}' ~ "'" }} - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' - access_log: /var/log/nginx/workbench2.fixme.example.net.access.log combined - error_log: /var/log/nginx/workbench2.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_workbench_configuration.sls b/test/salt/pillar/examples/nginx_workbench_configuration.sls index 34d8221..37fa31c 100644 --- a/test/salt/pillar/examples/nginx_workbench_configuration.sls +++ b/test/salt/pillar/examples/nginx_workbench_configuration.sls @@ -1,4 +1,8 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {%- if grains.os_family in ('RedHat',) %} {%- set group = 'nginx' %} {%- else %} @@ -32,6 +36,8 @@ nginx: arvados_workbench_ssl.conf: enabled: true overwrite: true + requires: + file: nginx_snippet_arvados-snakeoil.conf config: - server: - server_name: workbench.fixme.example.net @@ -41,8 +47,7 @@ nginx: - passenger_enabled: 'on' - index: index.html index.htm - include: 'snippets/ssl_hardening_default.conf' - # - include: 'snippets/letsencrypt.conf' - - include: 'snippets/ssl_snakeoil.conf' + - include: 'snippets/arvados-snakeoil.conf' # yamllint disable-line rule:line-length - access_log: /var/log/nginx/workbench.fixme.example.net.access.log combined - error_log: /var/log/nginx/workbench.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/postgresql.sls b/test/salt/pillar/examples/postgresql.sls index 5d800ec..0bf3357 100644 --- a/test/salt/pillar/examples/postgresql.sls +++ b/test/salt/pillar/examples/postgresql.sls @@ -1,9 +1,28 @@ --- +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + ### POSTGRESQL postgres: - use_upstream_repo: false + # Centos-7's postgres package is too old, so we need to force using upstream's + # This is not required in Debian's family as they already ship with PG +11 + {%- if salt['grains.get']('os_family') == 'RedHat' %} + use_upstream_repo: true + version: '12' + + pkgs_deps: + - libicu + - libxslt + - systemd-sysv + + pkgs_extra: + - postgresql12-contrib + + {%- else %} pkgs_extra: - postgresql-contrib + {%- endif %} postgresconf: |- listen_addresses = '*' # listen on all interfaces #ssl = on diff --git a/test/salt/states/examples/single_host/README.rst b/test/salt/states/examples/single_host/README.rst index b50716d..e23feab 100644 --- a/test/salt/states/examples/single_host/README.rst +++ b/test/salt/states/examples/single_host/README.rst @@ -1,6 +1,11 @@ Helper states for all-in-one setup ================================== +.. + # Copyright (C) The Arvados Authors. All rights reserved. + # + # SPDX-License-Identifier: Apache-2.0 + These states are helpful for setting up an all-in-one Arvados host. * `host_entries.sls`: adds a bunch of host entries in the `/etc/hosts` file of @@ -9,7 +14,7 @@ These states are helpful for setting up an all-in-one Arvados host. * `snakeoil_certs.sls`: Arvados uses SSL/TLS for communications, so you'll need certificates for the different hosts. If you can't provide valid certificates - issued by a recognized CA, this state will create a SnakeOil CA and issue + issued by a recognized CA, this state will create a SnakeOil CA and issue certificates signed by it. The certs can't be self-signed because some of the libraries that Arvados diff --git a/test/salt/states/examples/single_host/snakeoil_certs.sls b/test/salt/states/examples/single_host/snakeoil_certs.sls index 6ac8f87..91617e4 100644 --- a/test/salt/states/examples/single_host/snakeoil_certs.sls +++ b/test/salt/states/examples/single_host/snakeoil_certs.sls @@ -1,3 +1,7 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + {%- set curr_tpldir = tpldir %} {%- set tpldir = 'arvados' %} {%- from "arvados/map.jinja" import arvados with context %} @@ -8,9 +12,11 @@ include: - nginx.config - nginx.service -{%- set arvados_ca_cert_file = '/etc/ssl/certs/arvados-snakeoil-ca.pem' %} +# Debian uses different dirs for certs and keys, but being a Snake Oil example, +# we'll keep it simple here. +{%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %} {%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %} -{%- set arvados_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' %} +{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %} {%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %} {%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %} @@ -122,6 +128,9 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c - require: - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run + # We need this before we can add the nginx's snippet + - require_in: + - file: nginx_snippet_arvados-snakeoil.conf {%- if grains.get('os_family') == 'Debian' %} arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed: @@ -138,18 +147,6 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_c - require: - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed -{%- endif %} - -arvados_test_salt_states_examples_single_host_snakeoil_certs_nginx_snakeoil_file_managed: - file.managed: - - name: /etc/nginx/snippets/arvados-snakeoil.conf - - contents: | - ssl_certificate {{ arvados_cert_file }}; - ssl_certificate_key {{ arvados_key_file }}; - - watch_in: - - service: nginx_service - - require: - - pkg: passenger_install - - file: arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run - require_in: - - file: nginx_config + - file: nginx_snippet_arvados-snakeoil.conf +{%- endif %}