From 4bf9501a14f86845865244ee3ffb03a34707d36c Mon Sep 17 00:00:00 2001 From: =?utf8?q?Javier=20B=C3=A9rtoli?= Date: Sun, 21 Jun 2020 08:17:21 -0300 Subject: [PATCH] feat(components,version): add extra components, new version Features: * dispatcher: add component * shell: add webshell subcomponent * config: improve configuration management * config: add check pre-deploy * tests: add new tests * repo: allow to specify and use the dev or testing repos * ruby: add initial package management (mostly for centos) Fixes: * config: yaml_encode passwords, secrets and tokens * config: move some defaults to pillar * various random fixes Documentation: * examples: some fixes on nginx configs Tests: * dependencies: add locale formula and example * webshell: add webshell example, change localhost to 127.0.0.2 BREAKING CHANGES: Arvados was upgraded to a new version (2.1.0) so this formula focuses mostly in that version --- .salt-lint | 6 +- .travis.yml | 98 +++++++++-- arvados/api/package/clean.sls | 4 +- arvados/api/package/install.sls | 11 +- arvados/config/file.sls | 6 + arvados/config/init.sls | 1 + arvados/config/package/clean.sls | 10 ++ arvados/config/package/init.sls | 5 + arvados/config/package/install.sls | 11 ++ arvados/controller/package/install.sls | 1 + arvados/defaults.yaml | 65 +++++--- arvados/dispatcher/clean.sls | 6 + arvados/dispatcher/init.sls | 7 + arvados/dispatcher/package/clean.sls | 10 ++ arvados/dispatcher/package/init.sls | 5 + arvados/dispatcher/package/install.sls | 11 ++ arvados/dispatcher/service/clean.sls | 13 ++ arvados/dispatcher/service/init.sls | 5 + arvados/dispatcher/service/running.sls | 23 +++ arvados/files/default/config.tmpl.jinja | 155 +++++++++--------- arvados/init.sls | 3 + arvados/keepproxy/package/install.sls | 1 + arvados/keepstore/package/install.sls | 1 + arvados/keepweb/package/install.sls | 1 + arvados/osfamilymap.yaml | 20 +++ arvados/repo/clean.sls | 1 - arvados/repo/install.sls | 24 ++- arvados/ruby/clean.sls | 5 + arvados/ruby/init.sls | 5 + arvados/ruby/package/clean.sls | 28 ++++ arvados/ruby/package/init.sls | 5 + arvados/ruby/package/install.sls | 13 ++ arvados/shell/clean.sls | 2 + arvados/shell/config/clean.sls | 18 ++ arvados/shell/config/file.sls | 47 ++++++ .../default/shell-libpam-arvados.tmpl.jinja | 29 ++++ .../default/shell-shellinabox.tmpl.jinja | 10 ++ arvados/shell/config/init.sls | 5 + arvados/shell/init.sls | 2 + arvados/shell/package/clean.sls | 4 +- arvados/shell/package/init.sls | 1 + arvados/shell/package/install.sls | 38 ++++- arvados/shell/service/clean.sls | 13 ++ arvados/shell/service/init.sls | 5 + arvados/shell/service/running.sls | 19 +++ arvados/websocket/package/install.sls | 1 + arvados/workbench/package/clean.sls | 4 +- arvados/workbench/package/install.sls | 7 +- arvados/workbench2/package/install.sls | 1 + kitchen.yml | 97 ++++++++--- pillar.example | 90 ++++++++-- test/integration/api/controls/config_spec.rb | 2 +- .../controller/controls/config_spec.rb | 10 +- .../controller/controls/services_spec.rb | 7 +- .../keepproxy/controls/config_spec.rb | 10 +- .../keepstore/controls/config_spec.rb | 10 +- .../keepweb/controls/config_spec.rb | 6 +- test/integration/repo/controls/repo_spec.rb | 1 + .../integration/shell/controls/config_spec.rb | 49 ++++++ .../shell/controls/packages_spec.rb | 7 +- test/integration/shell/controls/repo_spec.rb | 20 +++ .../shell/controls/services_spec.rb | 16 ++ .../websocket/controls/config_spec.rb | 4 +- .../workbench/controls/config_spec.rb | 6 +- .../workbench2/controls/config_spec.rb | 2 +- test/salt/pillar/arvados.sls | 127 ++++++++++++++ test/salt/pillar/arvados_dev.sls | 153 +++++++++++++++++ test/salt/pillar/examples/locale.sls | 10 ++ .../examples/nginx_api_configuration.sls | 8 +- .../nginx_controller_configuration.sls | 15 +- .../nginx_keepproxy_configuration.sls | 12 +- .../examples/nginx_keepweb_configuration.sls | 12 +- test/salt/pillar/examples/nginx_passenger.sls | 5 - .../examples/nginx_webshell_configuration.sls | 71 ++++++++ .../nginx_websocket_configuration.sls | 12 +- .../nginx_workbench2_configuration.sls | 12 +- .../nginx_workbench_configuration.sls | 19 +-- test/salt/pillar/examples/postgresql.sls | 3 - 78 files changed, 1290 insertions(+), 272 deletions(-) create mode 100644 arvados/config/package/clean.sls create mode 100644 arvados/config/package/init.sls create mode 100644 arvados/config/package/install.sls create mode 100644 arvados/dispatcher/clean.sls create mode 100644 arvados/dispatcher/init.sls create mode 100644 arvados/dispatcher/package/clean.sls create mode 100644 arvados/dispatcher/package/init.sls create mode 100644 arvados/dispatcher/package/install.sls create mode 100644 arvados/dispatcher/service/clean.sls create mode 100644 arvados/dispatcher/service/init.sls create mode 100644 arvados/dispatcher/service/running.sls create mode 100644 arvados/ruby/clean.sls create mode 100644 arvados/ruby/init.sls create mode 100644 arvados/ruby/package/clean.sls create mode 100644 arvados/ruby/package/init.sls create mode 100644 arvados/ruby/package/install.sls create mode 100644 arvados/shell/config/clean.sls create mode 100644 arvados/shell/config/file.sls create mode 100644 arvados/shell/config/files/default/shell-libpam-arvados.tmpl.jinja create mode 100644 arvados/shell/config/files/default/shell-shellinabox.tmpl.jinja create mode 100644 arvados/shell/config/init.sls create mode 100644 arvados/shell/service/clean.sls create mode 100644 arvados/shell/service/init.sls create mode 100644 arvados/shell/service/running.sls create mode 100644 test/integration/shell/controls/config_spec.rb create mode 100644 test/integration/shell/controls/repo_spec.rb create mode 100644 test/integration/shell/controls/services_spec.rb create mode 100644 test/salt/pillar/arvados.sls create mode 100644 test/salt/pillar/arvados_dev.sls create mode 100644 test/salt/pillar/examples/locale.sls create mode 100644 test/salt/pillar/examples/nginx_webshell_configuration.sls diff --git a/.salt-lint b/.salt-lint index 3715677..2389f56 100644 --- a/.salt-lint +++ b/.salt-lint @@ -2,7 +2,11 @@ # vim: ft=yaml --- exclude_paths: [] -rules: {} +rules: + 204: # Lines should be no longer that 160 chars + ignore: | + arvados/shell/config/files/default/shell-libpam-arvados.tmpl.jinja + test/salt/pillar/examples/nginx_webshell_configuration.sls skip_list: # Using `salt-lint` for linting other files as well, such as Jinja macros/templates - 205 # Use ".sls" as a Salt State file extension diff --git a/.travis.yml b/.travis.yml index e7a4fd8..3ced9e0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -31,6 +31,9 @@ jobs: # Run all of the linters in a single job - language: 'node_js' node_js: 'lts/*' + cache: + directories: + - $HOME/.cache/pre-commit env: 'Lint' name: 'Lint: salt-lint, yamllint, rubocop, shellcheck & commitlint' before_install: 'skip' @@ -54,16 +57,87 @@ jobs: - npm i -D @commitlint/config-conventional @commitlint/travis-cli - commitlint-travis + # Install and run `pre-commit` + - pip install pre-commit + - pre-commit run --all-files --verbose + - pre-commit run --hook-stage manual --verbose commitlint-travis ## Define the rest of the matrix based on Kitchen testing # Make sure the instances listed below match up with # the `platforms` defined in `kitchen.yml` - - env: INSTANCE=api-debian-10-3000-3-py3 - - env: INSTANCE=workbench-debian-10-3000-3-py3 - - env: INSTANCE=shell-debian-10-3000-3-py3 - - env: INSTANCE=keepstore-debian-10-3000-3-py3 - # - env: INSTANCE=default-ubuntu-1804-3000-3-py3 - # - env: INSTANCE=default-centos-7-2019-2-py3 + + # - env: INSTANCE=api-debian-10-tiamat-py3 + # - env: INSTANCE=api-debian-9-tiamat-py3 + # - env: INSTANCE=api-ubuntu-2004-tiamat-py3 + # - env: INSTANCE=api-ubuntu-1804-tiamat-py3 + # - env: INSTANCE=api-centos-7-tiamat-py3 + # - env: INSTANCE=api-debian-10-master-py3 + # - env: INSTANCE=api-ubuntu-2004-master-py3 + # - env: INSTANCE=api-ubuntu-1804-master-py3 + - env: INSTANCE=api-debian-10-3001-py3 + # - env: INSTANCE=api-debian-9-3001-py3 + # - env: INSTANCE=api-ubuntu-2004-3001-py3 + # - env: INSTANCE=api-ubuntu-1804-3001-py3 + # - env: INSTANCE=api-centos-7-3001-py3 + # - env: INSTANCE=api-debian-10-3000-3-py3 + # - env: INSTANCE=api-debian-9-3000-3-py3 + # - env: INSTANCE=api-ubuntu-1804-3000-3-py3 + # - env: INSTANCE=api-centos-7-3000-3-py3 + # - env: INSTANCE=api-ubuntu-1804-3000-3-py2 + # - env: INSTANCE=workbench-debian-10-tiamat-py3 + # - env: INSTANCE=workbench-debian-9-tiamat-py3 + # - env: INSTANCE=workbench-ubuntu-2004-tiamat-py3 + # - env: INSTANCE=workbench-ubuntu-1804-tiamat-py3 + # - env: INSTANCE=workbench-centos-7-tiamat-py3 + # - env: INSTANCE=workbench-debian-10-master-py3 + # - env: INSTANCE=workbench-ubuntu-2004-master-py3 + - env: INSTANCE=workbench-ubuntu-1804-master-py3 + # - env: INSTANCE=workbench-debian-10-3001-py3 + # - env: INSTANCE=workbench-debian-9-3001-py3 + # - env: INSTANCE=workbench-ubuntu-2004-3001-py3 + # - env: INSTANCE=workbench-ubuntu-1804-3001-py3 + # - env: INSTANCE=workbench-centos-7-3001-py3 + # - env: INSTANCE=workbench-debian-10-3000-3-py3 + # - env: INSTANCE=workbench-debian-9-3000-3-py3 + # - env: INSTANCE=workbench-ubuntu-1804-3000-3-py3 + # - env: INSTANCE=workbench-centos-7-3000-3-py3 + # - env: INSTANCE=workbench-ubuntu-1804-3000-3-py2 + # - env: INSTANCE=shell-debian-10-tiamat-py3 + # - env: INSTANCE=shell-debian-9-tiamat-py3 + # - env: INSTANCE=shell-ubuntu-2004-tiamat-py3 + # - env: INSTANCE=shell-ubuntu-1804-tiamat-py3 + # - env: INSTANCE=shell-centos-7-tiamat-py3 + # - env: INSTANCE=shell-debian-10-master-py3 + # - env: INSTANCE=shell-ubuntu-2004-master-py3 + # - env: INSTANCE=shell-ubuntu-1804-master-py3 + - env: INSTANCE=shell-debian-10-3001-py3 + # - env: INSTANCE=shell-debian-9-3001-py3 + # - env: INSTANCE=shell-ubuntu-2004-3001-py3 + # - env: INSTANCE=shell-ubuntu-1804-3001-py3 + # - env: INSTANCE=shell-centos-7-3001-py3 + # - env: INSTANCE=shell-debian-10-3000-3-py3 + # - env: INSTANCE=shell-debian-9-3000-3-py3 + # - env: INSTANCE=shell-ubuntu-1804-3000-3-py3 + # - env: INSTANCE=shell-centos-7-3000-3-py3 + # - env: INSTANCE=shell-ubuntu-1804-3000-3-py2 + # - env: INSTANCE=keepstore-debian-10-tiamat-py3 + # - env: INSTANCE=keepstore-debian-9-tiamat-py3 + # - env: INSTANCE=keepstore-ubuntu-2004-tiamat-py3 + # - env: INSTANCE=keepstore-ubuntu-1804-tiamat-py3 + # - env: INSTANCE=keepstore-centos-7-tiamat-py3 + # - env: INSTANCE=keepstore-debian-10-master-py3 + # - env: INSTANCE=keepstore-ubuntu-2004-master-py3 + # - env: INSTANCE=keepstore-ubuntu-1804-master-py3 + # - env: INSTANCE=keepstore-debian-10-3001-py3 + # - env: INSTANCE=keepstore-debian-9-3001-py3 + # - env: INSTANCE=keepstore-ubuntu-2004-3001-py3 + # - env: INSTANCE=keepstore-ubuntu-1804-3001-py3 + # - env: INSTANCE=keepstore-centos-7-3001-py3 + # - env: INSTANCE=keepstore-debian-10-3000-3-py3 + # - env: INSTANCE=keepstore-debian-9-3000-3-py3 + # - env: INSTANCE=keepstore-ubuntu-1804-3000-3-py3 + # - env: INSTANCE=keepstore-centos-7-3000-3-py3 + - env: INSTANCE=keepstore-ubuntu-1804-3000-3-py2 ## Define the release stage that runs `semantic-release` - stage: 'release' @@ -90,15 +164,3 @@ jobs: edge: true # Run `semantic-release` script: 'npx semantic-release@15.14' - -# Notification options: `always`, `never` or `change` -notifications: - webhooks: - if: 'repo = saltstack-formulas/arvados-formula' - urls: - - https://saltstack-formulas.zulipchat.com/api/v1/external/travis?api_key=HsIq3o5QmLxdnVCKF9is0FUIpkpAY79P&stream=CI&topic=saltstack-formulas%2Farvados-formula&ignore_pull_requests=true - on_success: always # default: always - on_failure: always # default: always - on_start: always # default: never - on_cancel: always # default: always - on_error: always # default: always diff --git a/arvados/api/package/clean.sls b/arvados/api/package/clean.sls index f1b3288..74d807e 100644 --- a/arvados/api/package/clean.sls +++ b/arvados/api/package/clean.sls @@ -16,7 +16,7 @@ arvados-api-package-clean-gem-{{ gm }}-removed: arvados-api-package-clean-gems-deps-pkg-removed: pkg.removed: - pkgs: {{ arvados.ruby.gems_deps | json }} - - only_if: {{ arvados.ruby.manage_gems_deps }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" arvados-api-package-clean-pkg-removed: pkg.removed: @@ -25,4 +25,4 @@ arvados-api-package-clean-pkg-removed: arvados-api-package-clean-ruby-pkg-removed: pkg.removed: - name: {{ arvados.ruby.pkg }} - - only_if: {{ arvados.ruby.manage_ruby }} + - only_if: test "{{ arvados.ruby.manage_ruby | lower }}" = "true" diff --git a/arvados/api/package/install.sls b/arvados/api/package/install.sls index 5c48f0c..63cf427 100644 --- a/arvados/api/package/install.sls +++ b/arvados/api/package/install.sls @@ -13,14 +13,14 @@ include: arvados-api-package-install-ruby-pkg-installed: pkg.installed: - name: {{ arvados.ruby.pkg }} - - only_if: {{ arvados.ruby.manage_ruby }} + - only_if: test "{{ arvados.ruby.manage_ruby | lower }}" = "true" arvados-api-package-install-gems-deps-pkg-installed: pkg.installed: - - pkgs: {{ arvados.ruby.gems_deps | json }} - - only_if: {{ arvados.ruby.manage_gems_deps }} + - pkgs: {{ arvados.ruby.gems_deps | unique | json }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" -{% for gm in arvados.api.gem.name %} +{% for gm in arvados.api.gem.name | unique %} arvados-api-package-install-gem-{{ gm }}-installed: gem.installed: - name: {{ gm }} @@ -32,6 +32,7 @@ arvados-api-package-install-gem-{{ gm }}-installed: arvados-api-package-install-pkg-installed: pkg.installed: - - pkgs: {{ arvados.api.pkg.name | json }} + - name: {{ arvados.api.pkg.name }} + - version: {{ arvados.version }} - require: - sls: {{ sls_config_file }} diff --git a/arvados/config/file.sls b/arvados/config/file.sls index 9860797..5a9c1f0 100644 --- a/arvados/config/file.sls +++ b/arvados/config/file.sls @@ -6,6 +6,9 @@ {%- from tplroot ~ "/map.jinja" import arvados with context %} {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} +include: + - .package + arvados-config-file-file-managed: file.managed: - name: {{ arvados.config.file }} @@ -20,3 +23,6 @@ arvados-config-file-file-managed: - template: jinja - context: arvados: {{ arvados | json }} + - check_cmd: /usr/bin/arvados-server config-dump -config + - require: + - pkg: arvados-config-package-install-pkg-installed diff --git a/arvados/config/init.sls b/arvados/config/init.sls index 465ddfe..a41bca9 100644 --- a/arvados/config/init.sls +++ b/arvados/config/init.sls @@ -2,4 +2,5 @@ # vim: ft=sls include: + - .package - .file diff --git a/arvados/config/package/clean.sls b/arvados/config/package/clean.sls new file mode 100644 index 0000000..4d7effa --- /dev/null +++ b/arvados/config/package/clean.sls @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-config-package-clean-pkg-removed: + pkg.removed: + - name: arvados-server diff --git a/arvados/config/package/init.sls b/arvados/config/package/init.sls new file mode 100644 index 0000000..d3e5518 --- /dev/null +++ b/arvados/config/package/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .install diff --git a/arvados/config/package/install.sls b/arvados/config/package/install.sls new file mode 100644 index 0000000..031e3e8 --- /dev/null +++ b/arvados/config/package/install.sls @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-config-package-install-pkg-installed: + pkg.installed: + - name: arvados-server + - version: {{ arvados.version }} diff --git a/arvados/controller/package/install.sls b/arvados/controller/package/install.sls index adf40b3..de2166a 100644 --- a/arvados/controller/package/install.sls +++ b/arvados/controller/package/install.sls @@ -8,3 +8,4 @@ arvados-controller-package-install-pkg-installed: pkg.installed: - name: {{ arvados.controller.pkg.name }} + - version: {{ arvados.version }} diff --git a/arvados/defaults.yaml b/arvados/defaults.yaml index d040caf..a053b7f 100644 --- a/arvados/defaults.yaml +++ b/arvados/defaults.yaml @@ -2,7 +2,10 @@ # vim: ft=yaml --- arvados: - version: '2.0.2' + version: 'latest' + # Release: one of production, testing, development + # Used to select the repository to use + release: production # defaults to production use_upstream_repo: true repo: humanname: Arvados Official Repository @@ -10,6 +13,7 @@ arvados: manage_ruby: false pkg: ruby manage_gems_deps: false + gems_deps: - bundler - curl @@ -23,7 +27,7 @@ arvados: - libxml2 - libxml2-dev - make - - python-dev + - python3-dev - ruby-dev - zlib1g-dev @@ -33,7 +37,13 @@ arvados: group: root mode: 640 + # Experimental feature + # only available when 'release: development' + auto_reload_config: false + cluster: + force_legacy_api14: false + database: connection_pool_max: 32 @@ -42,12 +52,19 @@ arvados: key: '' insecure: false - ### SERVICES + Login: + # Google: + # ClientID: 94....apps.googleusercontent.com + # ClientSecret: FL....r + + PAM: + Enable: true + + ### THESE ARE THE PACKAGES AND DAEMONS BASIC CONFIGS ##### API api: pkg: - name: - - arvados-api-server + name: arvados-api-server gem: name: - arvados-cli @@ -64,20 +81,20 @@ arvados: ##### DISPATCHER dispatcher: pkg: - name: - - crunch-dispatch-local - # - arvados-dispatch-cloud - # - crunch-dispatch-slurm - service: name: crunch-dispatch-local - port: 9006 + # alternative dispatchers: + # arvados-dispatch-cloud + # crunch-dispatch-slurm + service: + name: '' + port: '' ##### KEEPPROXY keepproxy: pkg: name: keepproxy service: name: keepproxy - port: 25107 + port: 25100 ##### KEEPWEB keepweb: pkg: @@ -85,7 +102,7 @@ arvados: service: name: keep-web # webdav - port: 9002 + port: 9003 ##### KEEPSTORE keepstore: pkg: @@ -106,14 +123,23 @@ arvados: name: - arvados-client - arvados-src - - libpam-arvados - - python-arvados-fuse - - python-arvados-python-client + - libpam-arvados-go + - python3-arvados-fuse + - python3-arvados-python-client - python3-arvados-cwl-runner + - shellinabox gem: name: - arvados-cli - arvados-login-sync + shellinabox: + config: /etc/default/shellinabox + service: + name: shellinabox + port: 4200 + libpam_arvados: + config: /etc/pam.d/arvados + ##### WORKBENCH workbench: pkg: @@ -135,10 +161,3 @@ arvados: service: name: arvados-ws port: 8005 - ##### SSO - sso: - pkg: - name: arvados-sso - service: - name: arvados-sso - port: 8900 diff --git a/arvados/dispatcher/clean.sls b/arvados/dispatcher/clean.sls new file mode 100644 index 0000000..c176be2 --- /dev/null +++ b/arvados/dispatcher/clean.sls @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .service.clean + - .package.clean diff --git a/arvados/dispatcher/init.sls b/arvados/dispatcher/init.sls new file mode 100644 index 0000000..02a98b8 --- /dev/null +++ b/arvados/dispatcher/init.sls @@ -0,0 +1,7 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .package + - ..config + - .service diff --git a/arvados/dispatcher/package/clean.sls b/arvados/dispatcher/package/clean.sls new file mode 100644 index 0000000..6da51c2 --- /dev/null +++ b/arvados/dispatcher/package/clean.sls @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-dispatcher-package-clean-pkg-removed: + pkg.removed: + - name: {{ arvados.dispatcher.pkg.name }} diff --git a/arvados/dispatcher/package/init.sls b/arvados/dispatcher/package/init.sls new file mode 100644 index 0000000..d3e5518 --- /dev/null +++ b/arvados/dispatcher/package/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .install diff --git a/arvados/dispatcher/package/install.sls b/arvados/dispatcher/package/install.sls new file mode 100644 index 0000000..4c9c8ad --- /dev/null +++ b/arvados/dispatcher/package/install.sls @@ -0,0 +1,11 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-dispatcher-package-install-pkg-installed: + pkg.installed: + - name: {{ arvados.dispatcher.pkg.name }} + - version: {{ arvados.version }} diff --git a/arvados/dispatcher/service/clean.sls b/arvados/dispatcher/service/clean.sls new file mode 100644 index 0000000..7a3d663 --- /dev/null +++ b/arvados/dispatcher/service/clean.sls @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-dispatcher-service-clean-service-dead: + service.dead: + - name: {{ arvados.service.name }} + - enable: False + - require_in: + - pkg: arvados-dispatcher-package-clean-pkg-removed diff --git a/arvados/dispatcher/service/init.sls b/arvados/dispatcher/service/init.sls new file mode 100644 index 0000000..6fe4d1a --- /dev/null +++ b/arvados/dispatcher/service/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .running diff --git a/arvados/dispatcher/service/running.sls b/arvados/dispatcher/service/running.sls new file mode 100644 index 0000000..cda4e1a --- /dev/null +++ b/arvados/dispatcher/service/running.sls @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- set sls_config_file = tplroot ~ '.config.file' %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +{%- if arvados.dispatcher.pkg.name != 'crunch-dispatch-local' %} +include: + - ..package + - {{ sls_config_file }} + +arvados-dispatcher-service-running-service-running: + service.running: + - name: {{ arvados.dispatcher.service.name }} + - enable: true + - watch: + - sls: {{ sls_config_file }} + - require: + - pkg: arvados-dispatcher-package-install-pkg-installed + - only_if: test "{{ arvados.dispatcher.pkg.name }}" != "crunch-dispatch-local" +{%- endif %} diff --git a/arvados/files/default/config.tmpl.jinja b/arvados/files/default/config.tmpl.jinja index 31a9665..fe33a2a 100644 --- a/arvados/files/default/config.tmpl.jinja +++ b/arvados/files/default/config.tmpl.jinja @@ -3,113 +3,112 @@ # File managed by Salt at <{{ source }}>. # Your changes will be overwritten. # -# Please check https://doc.arvados.org/master/admin/config.html for +# Please check https://doc.arvados.org/master/admin/config.html for # documentation about the parameters configured here. ######################################################################## +{%- if arvados.release == 'development' %} +# (Experimental) Restart services automatically when config file +# changes are detected. Only supported by `arvados-server boot` in +# dev mode. +AutoReloadConfig: {{ arvados.auto_reload_config }} +{%- endif %} + Clusters: {{ arvados.cluster.name }}: - SystemRootToken: "{{ arvados.cluster.tokens.system_root }}" - ManagementToken: "{{ arvados.cluster.tokens.management }}" + + SystemRootToken: {{ arvados.cluster.tokens.system_root | yaml_encode }} + ManagementToken: {{ arvados.cluster.tokens.management | yaml_encode }} + + ForceLegacyAPI14: {{ arvados.cluster.force_legacy_api14 }} API: - RailsSessionSecretToken: "{{ arvados.cluster.tokens.rails_secret }}" + RailsSessionSecretToken: {{ arvados.cluster.tokens.rails_secret | yaml_encode }} + {%- if 'API' in arvados.cluster %} + {{ arvados.cluster.API | default('') | yaml(False) | indent(6) }} + {%- endif %} Collections: - BlobSigningKey: "{{ arvados.cluster.secrets.blob_signing_key }}" - ForwardSlashNameSubstitution: "%2f" - DefaultReplication: 1 - TrustAllContent: true + BlobSigningKey: {{ arvados.cluster.secrets.blob_signing_key | yaml_encode }} + {%- if 'Collections' in arvados.cluster %} + {{ arvados.cluster.Collections | yaml(False) | indent(6) }} + {%- endif %} Login: - ProviderAppSecret: "{{ arvados.cluster.tokens.provider_secret }}" - ProviderAppID: arvados-server + {{ arvados.cluster.Login | yaml(False) | indent(6) }} Users: - NewUsersAreActive: true - AutoAdminFirstUser: true - AutoSetupNewUsers: true - AutoSetupNewUsersWithVmUUID: x2jbo-2x53u-6maueyy9if4u7vq - AutoSetupNewUsersWithRepository: true + {{ arvados.cluster.Users | yaml(False) | indent(6) }} TLS: - Certificate: "{{ arvados.cluster.tls.certificate }}" - Key: "{{ arvados.cluster.tls.key }}" + Certificate: {{ arvados.cluster.tls.certificate | yaml_encode }} + Key: {{ arvados.cluster.tls.key | yaml_encode }} Insecure: {{ arvados.cluster.tls.insecure }} Workbench: - SecretKeyBase: {{ arvados.cluster.secrets.workbench_secret_key }} - SiteName: {{ arvados.cluster.name | upper }} - - # FIXME!!!! - # Git: - # GitCommand: /usr/share/gitolite3/gitolite-shell - # GitoliteHome: /var/lib/arvados/git - # Repositories: /var/lib/arvados/git/repositories + SecretKeyBase: {{ arvados.cluster.secrets.workbench_secret_key | yaml_encode }} + SiteName: {{ arvados.cluster.name | upper }} + {%- if 'Workbench' in arvados.cluster %} + {{ arvados.cluster.Workbench | yaml(False) | indent(6) }} + {%- endif %} - Volumes: - {% for v, p in arvados.cluster.volumes.items() -%} - ### {{ v | upper }} - {{ p.cluster }}-nyw5e-{{ p.volume_id }}: - Driver: {{ p.driver }} - DriverParameters: {{ p.driver_parameters | yaml }} - AccessViaHosts: {{ p.access_via_hosts | yaml }} - Replication: {{ p.replication }} - {% endfor -%} + {%- for section in [ + 'AuditLogs', + 'Containers', + 'Git', + 'InstanceTypes', + 'Mail', + 'RemoteClusters', + 'SystemLogs', + 'Volumes' + ] + %} + {%- if section in arvados.cluster %} + {{ section }}: + {{ arvados.cluster[section] | yaml(False) | indent(6) }} + {%- endif %} + {%- endfor %} ### DATABASE CONFIGURATION PostgreSQL: + # FIXME!!!!!! ALL as database or using Arvados' PostgreSQL ?? ConnectionPool: {{ arvados.cluster.database.connection_pool_max }} Connection: # All parameters here are passed to the PG client library in a connection string; # see https://www.postgresql.org/docs/current/static/libpq-connect.html#LIBPQ-PARAMKEYWORDS dbname: {{ arvados.cluster.database.name }} host: {{ arvados.cluster.database.host }} - password: {{ arvados.cluster.database.password }} + password: {{ arvados.cluster.database.password | yaml_encode }} user: {{ arvados.cluster.database.user }} client_encoding: {{ arvados.cluster.database.client_encoding }} + {%- if 'PostgreSQL' in arvados.cluster %} + {{ arvados.cluster.PostgreSQL | yaml(False) | indent(6) }} + {%- endif %} ### SERVICES URLs - # This could probably made into a loop, but some consistency check needs to be done - # on the arvados side before that's possible Services: - # Composer: FIXME!!! - Controller: - ExternalURL: "https://{{ arvados.cluster.domain }}" - InternalURLs: - "http://localhost:{{ arvados.controller.service.port }}": {} - DispatchCloud: - InternalURLs: - "http://localhost:{{ arvados.dispatcher.service.port }}": {} - # GitSSH: FIXME!!! - # Health: FIXME!!! - # Keepbalance: FIXME!!! - # Keepproxy: FIXME!!! - # Keepstore: FIXME!!! - # Nodemanager: FIXME!!! - RailsAPI: - InternalURLs: - "http://localhost:{{ arvados.api.service.port }}": {} - SSO: - ExternalURL: "https://sso.{{ arvados.cluster.domain }}" - WebDAV: - ExternalURL: https://collections.{{ arvados.cluster.domain }} - InternalURLs: - "http://localhost:{{ arvados.keepweb.service.port }}": {} - WebDAVDownload: - ExternalURL: https://download.{{ arvados.cluster.domain }} - # WebShell: FIXME!!! - Websocket: - ExternalURL: wss://ws.{{ arvados.cluster.domain }}/websocket - InternalURLs: - "http://localhost:{{ arvados.websocket.service.port }}": {} - Workbench1: - ExternalURL: "https://workbench.{{ arvados.cluster.domain }}" - Workbench2: - ExternalURL: "https://workbench2.{{ arvados.cluster.domain }}" - Keepproxy: - ExternalURL: https://keep.{{ arvados.cluster.domain }} - InternalURLs: - "http://localhost:{{ arvados.keepproxy.service.port }}": {} - Keepstore: - InternalURLs: - "http://keep0.{{ arvados.cluster.domain }}:{{ arvados.keepstore.service.port }}/": {} + {%- for service in [ + 'Composer', + 'Controller', + 'DispatchCloud', + 'GitHTTP', + 'GitSSH', + 'Health', + 'Keepbalance', + 'Keepproxy', + 'Keepstore', + 'Nodemanager', + 'RailsAPI', + 'SSO', + 'WebDAV', + 'WebDAVDownload', + 'WebShell', + 'Websocket', + 'Workbench1', + 'Workbench2', + ] + %} + {%- if service in arvados.cluster.Services %} + {{ service }}: + {{ arvados.cluster.Services[service] | yaml(False) | indent(8) }} + {%- endif %} + {%- endfor %} diff --git a/arvados/init.sls b/arvados/init.sls index ad7fdbb..9836070 100644 --- a/arvados/init.sls +++ b/arvados/init.sls @@ -4,7 +4,9 @@ include: - .repo - .config + - .ruby - .api + - .controller - .keepstore - .websocket - .keepweb @@ -12,3 +14,4 @@ include: - .keepproxy - .shell - .workbench + - .dispatcher diff --git a/arvados/keepproxy/package/install.sls b/arvados/keepproxy/package/install.sls index 1f1713f..e06faa6 100644 --- a/arvados/keepproxy/package/install.sls +++ b/arvados/keepproxy/package/install.sls @@ -8,3 +8,4 @@ arvados-keepproxy-package-install-pkg-installed: pkg.installed: - name: {{ arvados.keepproxy.pkg.name }} + - version: {{ arvados.version }} diff --git a/arvados/keepstore/package/install.sls b/arvados/keepstore/package/install.sls index 90220c7..9d311d5 100644 --- a/arvados/keepstore/package/install.sls +++ b/arvados/keepstore/package/install.sls @@ -8,3 +8,4 @@ arvados-keepstore-package-install-pkg-installed: pkg.installed: - name: {{ arvados.keepstore.pkg.name }} + - version: {{ arvados.version }} diff --git a/arvados/keepweb/package/install.sls b/arvados/keepweb/package/install.sls index c26164f..e9cfd4c 100644 --- a/arvados/keepweb/package/install.sls +++ b/arvados/keepweb/package/install.sls @@ -8,3 +8,4 @@ arvados-keepweb-package-install-pkg-installed: pkg.installed: - name: {{ arvados.keepweb.pkg.name }} + - version: {{ arvados.version }} diff --git a/arvados/osfamilymap.yaml b/arvados/osfamilymap.yaml index 7342b3d..ab316cb 100644 --- a/arvados/osfamilymap.yaml +++ b/arvados/osfamilymap.yaml @@ -21,3 +21,23 @@ RedHat: url_base: 'http://rpm.arvados.org/CentOS/$releasever/os/$basearch/' file: /etc/yum.repos.d/arvados.repo key_url: 'http://rpm.arvados.org/CentOS/RPM-GPG-KEY-curoverse' + + ruby: + manage_ruby: true + + pkg: ruby-2.5.7 + gems_deps: + - rubygem-bundler + - curl + - gcc + - git + - libcurl + - libcurl-devel + - pam-devel + - postgresql-devel + - libxml2 + - libxml2-devel + - make + - python3-devel + - ruby-devel + - zlib-devel diff --git a/arvados/repo/clean.sls b/arvados/repo/clean.sls index 85e6bcb..55cf601 100644 --- a/arvados/repo/clean.sls +++ b/arvados/repo/clean.sls @@ -16,7 +16,6 @@ arvados-repo-clean-repo-absent: arvados-repo-clean-repo-absent: pkgrepo.absent: - file: {{ arvados.repo.file }} - - gpgkey: {{ arvados.repo.gpgkey }} {%- else %} arvados_repo-clean-repo-absent: {} diff --git a/arvados/repo/install.sls b/arvados/repo/install.sls index 8478cd7..57d1ed2 100644 --- a/arvados/repo/install.sls +++ b/arvados/repo/install.sls @@ -5,24 +5,38 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -{% if arvados.use_upstream_repo -%} - {% if grains.get('os_family') == 'Debian' -%} +{%- if arvados.use_upstream_repo -%} + {%- if grains.get('os_family') == 'Debian' -%} + {%- if arvados.release == 'testing' %} + {%- set release = grains.get('lsb_distrib_codename') ~ '-testing' %} + {%- elif arvados.release == 'development' %} + {%- set release = grains.get('lsb_distrib_codename') ~ '-dev' %} + {%- else %} + {%- set release = grains.get('lsb_distrib_codename') %} + {%- endif %} arvados-repo-install-pkgrepo-managed: pkgrepo.managed: - humanname: {{ arvados.repo.humanname }} - - name: deb {{ arvados.repo.url_base }}/ {{ grains.get('lsb_distrib_codename') }} main + - name: deb {{ arvados.repo.url_base }}/ {{ release }} main - file: {{ arvados.repo.file }} - key_url: {{ arvados.repo.key_url }} {%- elif grains.get('os_family') == 'RedHat' %} + {%- if arvados.release == 'testing' %} + {%- set repo_url = 'http://rpm.arvados.org/CentOS/$releasever/testing/$basearch/' %} + {%- elif arvados.release == 'development' %} + {%- set repo_url = 'http://rpm.arvados.org/CentOS/$releasever/dev/$basearch/' %} + {%- else %} + {%- set repo_url = arvados.repo.url_base %} + {%- endif %} arvados-repo-install-pkgrepo-managed: pkgrepo.managed: - name: arvados - file: {{ arvados.repo.file }} - humanname: {{ arvados.repo.humanname }} - - baseurl: {{ arvados.repo.url_base }} + - baseurl: {{ repo_url }} - gpgcheck: 1 - - gpgkey: {{ arvados.repo.gpgkey }} + - gpgkey: {{ arvados.repo.key_url }} {%- else %} arvados-repo-install-pkgrepo-managed: {} diff --git a/arvados/ruby/clean.sls b/arvados/ruby/clean.sls new file mode 100644 index 0000000..f77852d --- /dev/null +++ b/arvados/ruby/clean.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .package.clean diff --git a/arvados/ruby/init.sls b/arvados/ruby/init.sls new file mode 100644 index 0000000..18fd41c --- /dev/null +++ b/arvados/ruby/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .package diff --git a/arvados/ruby/package/clean.sls b/arvados/ruby/package/clean.sls new file mode 100644 index 0000000..85941be --- /dev/null +++ b/arvados/ruby/package/clean.sls @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +{% for gm in arvados.shell.gem.name %} +arvados-shell-package-clean-gem-{{ gm }}-removed: + gem.removed: + - name: {{ gm }} + - require_in: + - pkg: arvados-shell-package-clean-gems-deps-pkg-removed +{% endfor %} + +arvados-shell-package-clean-gems-deps-pkg-removed: + pkg.removed: + - pkgs: {{ arvados.ruby.gems_deps | json }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" + +arvados-shell-package-clean-pkg-removed: + pkg.removed: + - pkgs: {{ arvados.shell.pkg.name | json }} + +arvados-shell-package-clean-ruby-pkg-removed: + pkg.removed: + - name: {{ arvados.ruby.pkg }} + - only_if: test "{{ arvados.ruby.manage_ruby | lower }}" = "true" diff --git a/arvados/ruby/package/init.sls b/arvados/ruby/package/init.sls new file mode 100644 index 0000000..d3e5518 --- /dev/null +++ b/arvados/ruby/package/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .install diff --git a/arvados/ruby/package/install.sls b/arvados/ruby/package/install.sls new file mode 100644 index 0000000..2200176 --- /dev/null +++ b/arvados/ruby/package/install.sls @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +{%- if arvados.ruby.manage_ruby %} +arvados-ruby-package-install-ruby-pkg-installed: + pkg.installed: + - name: {{ arvados.ruby.pkg }} + - require_in: arvados-ruby-package-install-ruby-profile-installed +{%- endif %} diff --git a/arvados/shell/clean.sls b/arvados/shell/clean.sls index f77852d..2cefe7d 100644 --- a/arvados/shell/clean.sls +++ b/arvados/shell/clean.sls @@ -2,4 +2,6 @@ # vim: ft=sls include: + - .service.clean + - .config.clean - .package.clean diff --git a/arvados/shell/config/clean.sls b/arvados/shell/config/clean.sls new file mode 100644 index 0000000..caceeca --- /dev/null +++ b/arvados/shell/config/clean.sls @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-shell-config-clean-file-shellinabox-absent: + file.absent: + - name: {{ arvados.shell.shellinabox.config }} + - watch_in: + - sls: {{ sls_service_clean }} + +arvados-shell-config-clean-file-libpam-arvados-absent: + file.absent: + - name: {{ arvados.shell.libpam-arvados.config }} + - watch_in: + - sls: {{ sls_service_clean }} diff --git a/arvados/shell/config/file.sls b/arvados/shell/config/file.sls new file mode 100644 index 0000000..c20ef99 --- /dev/null +++ b/arvados/shell/config/file.sls @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- set sls_package_install = tplroot ~ '.shell.package.install' %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} +{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} + +include: + - {{ sls_package_install }} + +arvados-shell-config-file-shellinabox-file-managed: + file.managed: + - name: {{ arvados.shell.shellinabox.config }} + - source: {{ files_switch(['shell-shellinabox.tmpl.jinja'], + lookup='arvados-shell-config-file-shellinabox-file-managed', + use_subpath=True + ) + }} + - mode: 644 + - user: root + - group: root + - makedirs: true + - template: jinja + - require: + - sls: {{ sls_package_install }} + - context: + arvados: {{ arvados | json }} + +arvados-shell-config-file-libpam-arvados-file-managed: + file.managed: + - name: {{ arvados.shell.libpam_arvados.config }} + - source: {{ files_switch(['shell-libpam-arvados.tmpl.jinja'], + lookup='arvados-shell-config-file-libpam-arvados-file-managed', + use_subpath=True + ) + }} + - mode: 644 + - user: root + - group: root + - makedirs: true + - template: jinja + - require: + - sls: {{ sls_package_install }} + - context: + arvados: {{ arvados | json }} diff --git a/arvados/shell/config/files/default/shell-libpam-arvados.tmpl.jinja b/arvados/shell/config/files/default/shell-libpam-arvados.tmpl.jinja new file mode 100644 index 0000000..1848e9d --- /dev/null +++ b/arvados/shell/config/files/default/shell-libpam-arvados.tmpl.jinja @@ -0,0 +1,29 @@ +######################################################################## +# File managed by Salt at <{{ source }}>. +# Your changes will be overwritten. +######################################################################## +auth optional pam_faildelay.so delay=3000000 +auth [success=ok new_authtok_reqd=ok ignore=ignore user_unknown=bad default=die] pam_securetty.so +auth requisite pam_nologin.so +session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so close +session required pam_env.so readenv=1 +session required pam_env.so readenv=1 envfile=/etc/default/locale + +# yamllint disable rule:line-length +auth [success=1 default=ignore] /usr/lib/pam_arvados.so {{ arvados.cluster.name }}.{{ arvados.cluster.domain }} shell.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }} +# yamllint enable rule:line-length +auth requisite pam_deny.so +auth required pam_permit.so + +auth optional pam_group.so +session required pam_limits.so +session optional pam_lastlog.so +session optional pam_motd.so motd=/run/motd.dynamic +session optional pam_motd.so +session optional pam_mail.so standard + +@include common-account +@include common-session +@include common-password + +session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so open diff --git a/arvados/shell/config/files/default/shell-shellinabox.tmpl.jinja b/arvados/shell/config/files/default/shell-shellinabox.tmpl.jinja new file mode 100644 index 0000000..6b0127f --- /dev/null +++ b/arvados/shell/config/files/default/shell-shellinabox.tmpl.jinja @@ -0,0 +1,10 @@ +######################################################################## +# File managed by Salt at <{{ source }}>. +# Your changes will be overwritten. +######################################################################## +# Should shellinaboxd start automatically +SHELLINABOX_DAEMON_START=1 +# TCP port that shellinboxd's webserver listens on +SHELLINABOX_PORT={{ arvados.shell.shellinabox.service.port }} +# SSL is disabled because it is terminated in Nginx. Adjust as needed. +SHELLINABOX_ARGS="--disable-ssl --no-beep --service=/shell.{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}:AUTH:HOME:SHELL" diff --git a/arvados/shell/config/init.sls b/arvados/shell/config/init.sls new file mode 100644 index 0000000..465ddfe --- /dev/null +++ b/arvados/shell/config/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .file diff --git a/arvados/shell/init.sls b/arvados/shell/init.sls index 18fd41c..858a8e6 100644 --- a/arvados/shell/init.sls +++ b/arvados/shell/init.sls @@ -3,3 +3,5 @@ include: - .package + - .config + - .service diff --git a/arvados/shell/package/clean.sls b/arvados/shell/package/clean.sls index edf6772..85941be 100644 --- a/arvados/shell/package/clean.sls +++ b/arvados/shell/package/clean.sls @@ -16,7 +16,7 @@ arvados-shell-package-clean-gem-{{ gm }}-removed: arvados-shell-package-clean-gems-deps-pkg-removed: pkg.removed: - pkgs: {{ arvados.ruby.gems_deps | json }} - - only_if: {{ arvados.ruby.manage_gems_deps }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" arvados-shell-package-clean-pkg-removed: pkg.removed: @@ -25,4 +25,4 @@ arvados-shell-package-clean-pkg-removed: arvados-shell-package-clean-ruby-pkg-removed: pkg.removed: - name: {{ arvados.ruby.pkg }} - - only_if: {{ arvados.ruby.manage_ruby }} + - only_if: test "{{ arvados.ruby.manage_ruby | lower }}" = "true" diff --git a/arvados/shell/package/init.sls b/arvados/shell/package/init.sls index d3e5518..bbf5da6 100644 --- a/arvados/shell/package/init.sls +++ b/arvados/shell/package/init.sls @@ -2,4 +2,5 @@ # vim: ft=sls include: + - ...ruby - .install diff --git a/arvados/shell/package/install.sls b/arvados/shell/package/install.sls index b9263f4..d2de401 100644 --- a/arvados/shell/package/install.sls +++ b/arvados/shell/package/install.sls @@ -3,21 +3,35 @@ {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} +{%- set sls_ruby_install = tplroot ~ '.ruby.package.install' %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -arvados-shell-package-install-pkg-installed: - pkg.installed: - - pkgs: {{ arvados.shell.pkg.name | json }} +include: + - {{ sls_ruby_install }} -arvados-shell-package-install-ruby-pkg-installed: +arvados-shell-package-install-pkg-installed: pkg.installed: - - name: {{ arvados.ruby.pkg }} - - only_if: {{ arvados.ruby.manage_ruby }} + - pkgs: + {%- for package in arvados.shell.pkg.name %} + # We use version for our Arvados packages only + {%- if package in [ + 'arvados-client', + 'arvados-src', + 'libpam-arvados-go', + 'python3-arvados-fuse', + 'python3-arvados-python-client', + 'python3-arvados-cwl-runner', + ] %} + - {{ package }}: {{ arvados.version }} + {%- else %} + - {{ package }} + {%- endif %} + {%- endfor %} arvados-shell-package-install-gems-deps-pkg-installed: pkg.installed: - pkgs: {{ arvados.ruby.gems_deps | json }} - - only_if: {{ arvados.ruby.manage_gems_deps }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" {% for gm in arvados.shell.gem.name %} arvados-shell-package-install-gem-{{ gm }}-installed: @@ -25,4 +39,14 @@ arvados-shell-package-install-gem-{{ gm }}-installed: - name: {{ gm }} - require: - pkg: arvados-shell-package-install-gems-deps-pkg-installed + {%- if arvados.ruby.manage_ruby %} + {%- if salt['grains.get']('osfinger') != 'CentOS Linux-7' %} + - pkg: arvados-ruby-package-install-ruby-pkg-installed + {%- else %} + # - rvm: arvados-ruby-package-install-ruby-gemset-present + - rvm: arvados-ruby-package-install-ruby-rvm-installed + # - rvm: gemset_present + # - ruby: ruby-2.5.7@arvados + {%- endif %} + {%- endif %} {% endfor %} diff --git a/arvados/shell/service/clean.sls b/arvados/shell/service/clean.sls new file mode 100644 index 0000000..cb3d932 --- /dev/null +++ b/arvados/shell/service/clean.sls @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +arvados-shell-shellinabox-service-clean-service-dead: + service.dead: + - name: {{ arvados.service.name }} + - enable: False + - require_in: + - pkg: arvados-shell-package-clean-pkg-removed diff --git a/arvados/shell/service/init.sls b/arvados/shell/service/init.sls new file mode 100644 index 0000000..6fe4d1a --- /dev/null +++ b/arvados/shell/service/init.sls @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +include: + - .running diff --git a/arvados/shell/service/running.sls b/arvados/shell/service/running.sls new file mode 100644 index 0000000..b70278d --- /dev/null +++ b/arvados/shell/service/running.sls @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} + +include: + - ..package + - ..config + +arvados-shell-shellinabox-service-running-service-running: + service.running: + - name: {{ arvados.shell.shellinabox.service.name }} + - enable: True + - watch: + - file: arvados-shell-config-file-shellinabox-file-managed + - require: + - pkg: arvados-shell-package-install-pkg-installed diff --git a/arvados/websocket/package/install.sls b/arvados/websocket/package/install.sls index 94cebc2..fe8f87a 100644 --- a/arvados/websocket/package/install.sls +++ b/arvados/websocket/package/install.sls @@ -8,3 +8,4 @@ arvados-websocket-package-install-pkg-installed: pkg.installed: - name: {{ arvados.websocket.pkg.name }} + - version: {{ arvados.version }} diff --git a/arvados/workbench/package/clean.sls b/arvados/workbench/package/clean.sls index 3d7af2f..fcfd2ba 100644 --- a/arvados/workbench/package/clean.sls +++ b/arvados/workbench/package/clean.sls @@ -8,7 +8,7 @@ arvados-workbench-package-clean-gems-deps-pkg-removed: pkg.removed: - pkgs: {{ arvados.ruby.gems_deps | json }} - - only_if: {{ arvados.ruby.manage_gems_deps }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" arvados-workbench-package-clean-pkg-removed: pkg.removed: @@ -17,4 +17,4 @@ arvados-workbench-package-clean-pkg-removed: arvados-workbench-package-clean-ruby-pkg-removed: pkg.removed: - name: {{ arvados.ruby.pkg }} - - only_if: {{ arvados.ruby.manage_ruby }} + - only_if: test "{{ arvados.ruby.manage_ruby | lower }}" = "true" diff --git a/arvados/workbench/package/install.sls b/arvados/workbench/package/install.sls index e6deefe..7487a60 100644 --- a/arvados/workbench/package/install.sls +++ b/arvados/workbench/package/install.sls @@ -6,22 +6,23 @@ {%- set sls_config_file = tplroot ~ '.config.file' %} {%- from tplroot ~ "/map.jinja" import arvados with context %} -# The API server requires a valid config BEFORE installing... +# The workbench server requires a valid config BEFORE installing... include: - {{ sls_config_file }} arvados-workbench-package-install-ruby-pkg-installed: pkg.installed: - name: {{ arvados.ruby.pkg }} - - only_if: {{ arvados.ruby.manage_ruby }} + - only_if: test "{{ arvados.ruby.manage_ruby | lower }}" = "true" arvados-workbench-package-install-gems-deps-pkg-installed: pkg.installed: - pkgs: {{ arvados.ruby.gems_deps | json }} - - only_if: {{ arvados.ruby.manage_gems_deps }} + - only_if: test "{{ arvados.ruby.manage_gems_deps | lower }}" = "true" arvados-workbench-package-install-pkg-installed: pkg.installed: - name: {{ arvados.workbench.pkg.name }} + - version: {{ arvados.version }} - require: - sls: {{ sls_config_file }} diff --git a/arvados/workbench2/package/install.sls b/arvados/workbench2/package/install.sls index 8aa27e6..9e503ff 100644 --- a/arvados/workbench2/package/install.sls +++ b/arvados/workbench2/package/install.sls @@ -9,5 +9,6 @@ arvados-workbench2-package-install-pkg-installed: pkg.installed: - name: {{ arvados.workbench2.pkg.name }} + - version: {{ arvados.version }} - require: - sls: {{ sls_config_file }} diff --git a/kitchen.yml b/kitchen.yml index 637955e..b55392b 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -11,19 +11,67 @@ driver: # Make sure the platforms listed below match up with # the `env.matrix` instances defined in `.travis.yml` platforms: + ## SALT `tiamat` + - name: debian-10-tiamat-py3 + driver: + image: saltimages/salt-tiamat-py3:debian-10 + - name: debian-9-tiamat-py3 + driver: + image: saltimages/salt-tiamat-py3:debian-9 + - name: ubuntu-2004-tiamat-py3 + driver: + image: saltimages/salt-tiamat-py3:ubuntu-20.04 + - name: ubuntu-1804-tiamat-py3 + driver: + image: saltimages/salt-tiamat-py3:ubuntu-18.04 + - name: centos-7-tiamat-py3 + driver: + image: saltimages/salt-tiamat-py3:centos-7 + + ## SALT `master` + - name: debian-10-master-py3 + driver: + image: saltimages/salt-master-py3:debian-10 + - name: ubuntu-2004-master-py3 + driver: + image: saltimages/salt-master-py3:ubuntu-20.04 + - name: ubuntu-1804-master-py3 + driver: + image: saltimages/salt-master-py3:ubuntu-18.04 + + ## SALT `3001` + - name: debian-10-3001-py3 + driver: + image: saltimages/salt-3001-py3:debian-10 + - name: debian-9-3001-py3 + driver: + image: saltimages/salt-3001-py3:debian-9 + - name: ubuntu-2004-3001-py3 + driver: + image: saltimages/salt-3001-py3:ubuntu-20.04 + - name: ubuntu-1804-3001-py3 + driver: + image: saltimages/salt-3001-py3:ubuntu-18.04 + - name: centos-7-3001-py3 + driver: + image: saltimages/salt-3001-py3:centos-7 + ## SALT `3000.3` - name: debian-10-3000-3-py3 driver: image: saltimages/salt-3000.3-py3:debian-10 - -# - name: ubuntu-1804-3000-3-py3 -# driver: -# image: saltimages/salt-3000.3-py3:ubuntu-18.04 - -# ## SALT `2019.2` -# - name: centos-7-2019-2-py3 -# driver: -# image: saltimages/salt-2019.2-py3:centos-7 + - name: debian-9-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:debian-9 + - name: ubuntu-1804-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:ubuntu-18.04 + - name: centos-7-3000-3-py3 + driver: + image: saltimages/salt-3000.3-py3:centos-7 + - name: ubuntu-1804-3000-3-py2 + driver: + image: saltimages/salt-3000.3-py2:ubuntu-18.04 provisioner: name: salt_solo @@ -47,15 +95,18 @@ suites: #### api,websocket,keepproxy,keepweb,controller - name: api driver: - hostname: example.net + hostname: fixme.example.net provisioner: state_top: base: '*': - example_add_snakeoil_certs + - locale - nginx.passenger - postgres - arvados.repo + # - arvados.config + # - arvados.ruby - arvados.api - arvados.websocket # keepproxy complains when using snakeoil certs, so we can't @@ -63,11 +114,13 @@ suites: # - arvados.keepproxy - arvados.keepweb - arvados.controller + - arvados.dispatcher pillars: top.sls: base: '*': - arvados + - example_locale - example_postgres - example_nginx - example_nginx_api @@ -79,7 +132,8 @@ suites: # - example_nginx_keepproxy pillars_from_files: # yamllint disable rule:line-length - arvados.sls: pillar.example + arvados.sls: test/salt/pillar/arvados.sls + example_locale.sls: test/salt/pillar/examples/locale.sls example_postgres.sls: test/salt/pillar/examples/postgresql.sls example_nginx.sls: test/salt/pillar/examples/nginx_passenger.sls example_nginx_api.sls: test/salt/pillar/examples/nginx_api_configuration.sls @@ -91,12 +145,15 @@ suites: dependencies: - name: example_add_snakeoil_certs path: test/salt/states + - name: locale + repo: git + source: https://github.com/saltstack-formulas/locale-formula.git - name: postgres repo: git source: https://github.com/saltstack-formulas/postgres-formula.git - name: nginx repo: git - source: https://github.com/saltstack-formulas/nginx-formula.git + source: https://github.com/netmanagers/nginx-formula.git verifier: inspec_tests: - path: test/integration/api @@ -108,7 +165,7 @@ suites: #### workbench,workbench2 - name: workbench driver: - hostname: workbench.example.net + hostname: workbench.fixme.example.net provisioner: state_top: base: @@ -128,7 +185,7 @@ suites: - example_nginx_workbench2 pillars_from_files: # yamllint disable rule:line-length - arvados.sls: pillar.example + arvados.sls: test/salt/pillar/arvados.sls example_nginx.sls: test/salt/pillar/examples/nginx_passenger.sls example_nginx_workbench.sls: test/salt/pillar/examples/nginx_workbench_configuration.sls example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls @@ -147,7 +204,7 @@ suites: #### shell - name: shell driver: - hostname: shell.example.net + hostname: shell.fixme.example.net provisioner: state_top: base: @@ -161,22 +218,21 @@ suites: '*': - arvados pillars_from_files: - arvados.sls: pillar.example + arvados.sls: test/salt/pillar/arvados_dev.sls verifier: inspec_tests: - - path: test/integration/repo - path: test/integration/shell #### keepstore - name: keepstore driver: - hostname: keep0.example.net + hostname: keep0.fixme.example.net provisioner: state_top: base: '*': - arvados.repo - - arvados.keepstore.service + - arvados.keepstore # - arvados.clean pillars: top.sls: @@ -184,7 +240,8 @@ suites: '*': - arvados pillars_from_files: - arvados.sls: pillar.example + arvados.sls: test/salt/pillar/arvados.sls verifier: inspec_tests: + - path: test/integration/repo - path: test/integration/keepstore diff --git a/pillar.example b/pillar.example index c3f669d..7bdfe98 100644 --- a/pillar.example +++ b/pillar.example @@ -6,7 +6,8 @@ # this formula will fail. arvados: ### GENERAL CONFIG - # version: '2.0.2' + # version: '2.1.0' + # release: production ## It makes little sense to disable this flag, but you can, if you want :) # use_upstream_repo: true @@ -38,7 +39,7 @@ arvados: # - libxml2 # - libxml2-dev # - make - # - python-dev + # - python3-dev # - ruby-dev # - zlib1g-dev @@ -78,7 +79,6 @@ arvados: management: changeme_management_token rails_secret: changeme_rails_secret_token anonymous_user: changeme_anonymous_user_token - provider_secret: changeme_provider_secret_token ### KEYS secrets: @@ -89,22 +89,69 @@ arvados: keep_access_key: changeme_keep_access_key keep_secret_key: changeme_keep_secret_key + AuditLogs: + Section_to_ignore: + - some_random_value + ### VOLUMES ## This should usually match all your `keepstore` instances - volumes: - volume_one: - # the volume name will be composed with - # -nyw5e- - cluster: fixme - volume_id: '000000000000000' - access_via_hosts: - "http://keep0.example.net:25107/": {} - replication: 2 - driver: Directory - driver_parameters: + Volumes: + # the volume name will be composed with + # -nyw5e- + fixme-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.fixme.example.net:25107: + ReadOnly: false + Replication: 2 + Driver: Directory + DriverParameters: Root: /tmp -# ## SERVICES + Users: + NewUsersAreActive: true + AutoAdminFirstUser: true + AutoSetupNewUsers: true + AutoSetupNewUsersWithRepository: true + + Services: + Controller: + ExternalURL: https://fixme.example.net + InternalURLs: + http://localhost:8003: {} + DispatchCloud: + InternalURLs: + http://fixme.example.net:9006: {} + Keepbalance: + InternalURLs: + http://fixme.example.net:9005: {} + Keepproxy: + ExternalURL: https://keep.fixme.example.net + InternalURLs: + http://localhost:25100: {} + Keepstore: + InternalURLs: + http://keep0.fixme.example.net:25107: {} + RailsAPI: + InternalURLs: + http://localhost:8004: {} + WebDAV: + ExternalURL: https://collections.fixme.example.net + InternalURLs: + http://localhost:9002: {} + WebDAVDownload: + ExternalURL: https://download.fixme.example.net + WebShell: + ExternalURL: https://webshell.fixme.example.net + Websocket: + ExternalURL: wss://ws.fixme.example.net/websocket + InternalURLs: + http://localhost:8005: {} + Workbench1: + ExternalURL: https://workbench.fixme.example.net + Workbench2: + ExternalURL: https://workbench2.fixme.example.net + +# ### THESE ARE THE PACKAGES AND DAEMONS BASIC CONFIGS # #### API # api: # pkg: @@ -170,14 +217,21 @@ arvados: # name: # - arvados-client # - arvados-src -# - libpam-arvados -# - python-arvados-fuse -# - python-arvados-python-client +# - libpam-arvados-go +# - python3-arvados-fuse +# - python3-arvados-python-client # - python3-arvados-cwl-runner # gem: # name: # - arvados-cli # - arvados-login-sync +# shellinabox: +# config: /etc/default/shellinabox +# service: +# name: shellinabox +# port: 4200 +# libpam_arvados: +# config: /etc/pam.d/arvados # #### WORKBENCH # workbench: # pkg: diff --git a/test/integration/api/controls/config_spec.rb b/test/integration/api/controls/config_spec.rb index 8efe253..7ad922f 100644 --- a/test/integration/api/controls/config_spec.rb +++ b/test/integration/api/controls/config_spec.rb @@ -8,7 +8,7 @@ API_STANZA rails_stanza = <<-RAILS_STANZA RailsAPI: InternalURLs: - "http://localhost:8004": {} + http://127.0.0.2:8004: {} RAILS_STANZA control 'arvados configuration' do diff --git a/test/integration/controller/controls/config_spec.rb b/test/integration/controller/controls/config_spec.rb index 6e64bc4..c3bc176 100644 --- a/test/integration/controller/controls/config_spec.rb +++ b/test/integration/controller/controls/config_spec.rb @@ -3,16 +3,18 @@ controller_stanza = <<-CONTROLLER_STANZA Keepstore: InternalURLs: - "http://keep0.example.net:25107/": {} + http://keep0.fixme.example.net:25107: {} CONTROLLER_STANZA volumes_stanza = <<-VOLUMES_STANZA Volumes: - ### VOLUME_ONE fixme-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.fixme.example.net:25107: + ReadOnly: false Driver: Directory - DriverParameters: {Root: /tmp} - AccessViaHosts: {'http://keep0.example.net:25107/': {}} + DriverParameters: + Root: /tmp Replication: 2 VOLUMES_STANZA diff --git a/test/integration/controller/controls/services_spec.rb b/test/integration/controller/controls/services_spec.rb index abee521..3b24532 100644 --- a/test/integration/controller/controls/services_spec.rb +++ b/test/integration/controller/controls/services_spec.rb @@ -4,12 +4,13 @@ control 'arvados controller service' do impact 0.5 title 'should be running and enabled' - describe service('keep-web') do + describe service('arvados-controller') do it { should be_enabled } it { should be_running } end - describe port(9002) do + describe port(8003) do it { should be_listening } - its('processes') { should include 'keep-web' } + # The undelying tools inspec uses to get the process truncates their names + its('processes') { should include 'arvados-control' } end end diff --git a/test/integration/keepproxy/controls/config_spec.rb b/test/integration/keepproxy/controls/config_spec.rb index 30feed4..7dd4f71 100644 --- a/test/integration/keepproxy/controls/config_spec.rb +++ b/test/integration/keepproxy/controls/config_spec.rb @@ -3,16 +3,18 @@ keepproxy_stanza = <<-KEEPPROXY_STANZA Keepstore: InternalURLs: - "http://keep0.example.net:25107/": {} + http://keep0.fixme.example.net:25107: {} KEEPPROXY_STANZA volumes_stanza = <<-VOLUMES_STANZA Volumes: - ### VOLUME_ONE fixme-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.fixme.example.net:25107: + ReadOnly: false Driver: Directory - DriverParameters: {Root: /tmp} - AccessViaHosts: {'http://keep0.example.net:25107/': {}} + DriverParameters: + Root: /tmp Replication: 2 VOLUMES_STANZA diff --git a/test/integration/keepstore/controls/config_spec.rb b/test/integration/keepstore/controls/config_spec.rb index 9fb74c9..d44e673 100644 --- a/test/integration/keepstore/controls/config_spec.rb +++ b/test/integration/keepstore/controls/config_spec.rb @@ -3,16 +3,18 @@ keepstore_stanza = <<-KEEPSTORE_STANZA Keepstore: InternalURLs: - "http://keep0.example.net:25107/": {} + http://keep0.fixme.example.net:25107: {} KEEPSTORE_STANZA volumes_stanza = <<-VOLUMES_STANZA Volumes: - ### VOLUME_ONE fixme-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.fixme.example.net:25107: + ReadOnly: false Driver: Directory - DriverParameters: {Root: /tmp} - AccessViaHosts: {'http://keep0.example.net:25107/': {}} + DriverParameters: + Root: /tmp Replication: 2 VOLUMES_STANZA diff --git a/test/integration/keepweb/controls/config_spec.rb b/test/integration/keepweb/controls/config_spec.rb index c678dcc..c2bd8e9 100644 --- a/test/integration/keepweb/controls/config_spec.rb +++ b/test/integration/keepweb/controls/config_spec.rb @@ -2,11 +2,11 @@ keepweb_stanza = <<-KEEPWEB_STANZA WebDAV: - ExternalURL: https://collections.example.net + ExternalURL: https://collections.fixme.example.net InternalURLs: - "http://localhost:9002": {} + http://127.0.0.2:9002: {} WebDAVDownload: - ExternalURL: https://download.example.net + ExternalURL: https://download.fixme.example.net KEEPWEB_STANZA control 'arvados configuration' do diff --git a/test/integration/repo/controls/repo_spec.rb b/test/integration/repo/controls/repo_spec.rb index 27cf73f..bfde111 100644 --- a/test/integration/repo/controls/repo_spec.rb +++ b/test/integration/repo/controls/repo_spec.rb @@ -3,6 +3,7 @@ case os[:name] when 'centos' repo_file = '/etc/yum.repos.d/arvados.repo' + repo_url = 'baseurl=http://rpm.arvados.org/CentOS/$releasever/os/$basearch/' when 'debian' repo_file = '/etc/apt/sources.list.d/arvados.list' diff --git a/test/integration/shell/controls/config_spec.rb b/test/integration/shell/controls/config_spec.rb new file mode 100644 index 0000000..9fa65b7 --- /dev/null +++ b/test/integration/shell/controls/config_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +siab_stanza = <<~SIAB_STANZA + SHELLINABOX_PORT=4200 + # SSL is disabled because it is terminated in Nginx. Adjust as needed. + SHELLINABOX_ARGS="--disable-ssl --no-beep --service=/shell.fixme.example.net:AUTH:HOME:SHELL" +SIAB_STANZA + +libpam_stanza = <<~LIBPAM_STANZA + auth [success=1 default=ignore] /usr/lib/pam_arvados.so fixme.example.net shell.fixme.example.net +LIBPAM_STANZA + +control 'shellinabox configuration' do + title 'should match desired lines' + + describe file('/etc/default/shellinabox') do + it { should be_file } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } + its('mode') { should cmp '0644' } + its('content') do + should include( + # rubocop:disable Metrics/LineLength + 'File managed by Salt at .' + # rubocop:enable Metrics/LineLength + ) + end + its('content') { should include(siab_stanza) } + end +end + +control 'libpam-arvados configuration' do + title 'should match desired lines' + + describe file('/etc/pam.d/arvados') do + it { should be_file } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } + its('mode') { should cmp '0644' } + its('content') do + should include( + # rubocop:disable Metrics/LineLength + 'File managed by Salt at .' + # rubocop:enable Metrics/LineLength + ) + end + its('content') { should include(libpam_stanza) } + end +end diff --git a/test/integration/shell/controls/packages_spec.rb b/test/integration/shell/controls/packages_spec.rb index 080095d..0b69de9 100644 --- a/test/integration/shell/controls/packages_spec.rb +++ b/test/integration/shell/controls/packages_spec.rb @@ -3,10 +3,11 @@ packages_list = %w[ arvados-client arvados-src - libpam-arvados - python-arvados-fuse - python-arvados-python-client + libpam-arvados-go + python3-arvados-fuse + python3-arvados-python-client python3-arvados-cwl-runner + shellinabox ] gems_list = %w[ diff --git a/test/integration/shell/controls/repo_spec.rb b/test/integration/shell/controls/repo_spec.rb new file mode 100644 index 0000000..10d1929 --- /dev/null +++ b/test/integration/shell/controls/repo_spec.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +case os[:name] +when 'centos' + repo_file = '/etc/yum.repos.d/arvados.repo' + + repo_url = 'baseurl=http://rpm.arvados.org/CentOS/$releasever/dev/$basearch/' +when 'debian' + repo_file = '/etc/apt/sources.list.d/arvados.list' + repo_url = 'deb http://apt.arvados.org buster-dev main' +end + +control 'arvados repository' do + impact 1 + title 'should be configured' + desc 'Ensures arvados source is correctly configured' + describe file(repo_file) do + its('content') { should match repo_url } + end +end diff --git a/test/integration/shell/controls/services_spec.rb b/test/integration/shell/controls/services_spec.rb new file mode 100644 index 0000000..a33f268 --- /dev/null +++ b/test/integration/shell/controls/services_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +control 'arvados shellinabox service' do + impact 0.5 + title 'should be running and enabled' + + describe service('shellinabox') do + it { should be_enabled } + it { should be_running } + end + + describe port(4200) do + it { should be_listening } + its('processes') { should include 'shellinaboxd' } + end +end diff --git a/test/integration/websocket/controls/config_spec.rb b/test/integration/websocket/controls/config_spec.rb index 57c237b..e4a4779 100644 --- a/test/integration/websocket/controls/config_spec.rb +++ b/test/integration/websocket/controls/config_spec.rb @@ -2,9 +2,9 @@ websocket_stanza = <<-WEBSOCKET_STANZA Websocket: - ExternalURL: wss://ws.example.net/websocket + ExternalURL: wss://ws.fixme.example.net/websocket InternalURLs: - "http://localhost:8005": {} + http://127.0.0.2:8005: {} WEBSOCKET_STANZA control 'arvados configuration' do diff --git a/test/integration/workbench/controls/config_spec.rb b/test/integration/workbench/controls/config_spec.rb index 7a2de62..ddcb550 100644 --- a/test/integration/workbench/controls/config_spec.rb +++ b/test/integration/workbench/controls/config_spec.rb @@ -2,13 +2,13 @@ workbench_config = <<-WORKBENCH_STANZA Workbench: - SecretKeyBase: changeme_workbench_secret_key - SiteName: FIXME + SecretKeyBase: "changeme_workbench_secret_key" + SiteName: FIXME WORKBENCH_STANZA workbench_service = <<-WORKBENCH_SERVICE_STANZA Workbench1: - ExternalURL: "https://workbench.example.net" + ExternalURL: https://workbench.fixme.example.net WORKBENCH_SERVICE_STANZA control 'arvados configuration' do diff --git a/test/integration/workbench2/controls/config_spec.rb b/test/integration/workbench2/controls/config_spec.rb index a6da207..5583688 100644 --- a/test/integration/workbench2/controls/config_spec.rb +++ b/test/integration/workbench2/controls/config_spec.rb @@ -2,7 +2,7 @@ workbench2_service = <<-WORKBENCH2_STANZA Workbench2: - ExternalURL: "https://workbench2.example.net" + ExternalURL: https://workbench2.fixme.example.net WORKBENCH2_STANZA control 'arvados configuration' do diff --git a/test/salt/pillar/arvados.sls b/test/salt/pillar/arvados.sls new file mode 100644 index 0000000..13a9dbd --- /dev/null +++ b/test/salt/pillar/arvados.sls @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# The variables commented out are the default values that the formula uses. +# The uncommented values are REQUIRED values. If you don't set them, running +# this formula will fail. +arvados: + ### GENERAL CONFIG + # version: '2.0.4' + ## It makes little sense to disable this flag, but you can, if you want :) + # use_upstream_repo: true + + ## Repo URL is built with grains values. If desired, it can be completely + ## overwritten with the pillar parameter 'repo_url' + # repo: + # humanname: Arvados Official Repository + + ## IMPORTANT!!!!! + ## api, workbench and shell require some gems, so you need to make sure ruby + ## and deps are installed in order to install and compile the gems. + ## We default to `false` in these two variables as it's expected you already + ## manage OS packages with some other tool and you don't want us messing up + ## with your setup. + ruby: + ## We set these to `true` here for testing purposes. + ## They both default to `false`. + manage_ruby: true + manage_gems_deps: true + + ### ARVADOS CLUSTER CONFIG + cluster: + name: fixme + domain: example.net + + database: + # max concurrent connections per arvados server daemon + # connection_pool_max: 32 + name: arvados + host: 127.0.0.1 + password: changeme_arvados + user: arvados + encoding: en_US.utf8 + client_encoding: UTF8 + + tls: + # certificate: '' + # key: '' + # required to test with snakeoil certs + insecure: true + + ### TOKENS + tokens: + system_root: changeme_system_root_token + management: changeme_management_token + rails_secret: changeme_rails_secret_token + anonymous_user: changeme_anonymous_user_token + + ### KEYS + secrets: + blob_signing_key: changeme_blob_signing_key + workbench_secret_key: changeme_workbench_secret_key + dispatcher_access_key: changeme_dispatcher_access_key + dispatcher_secret_key: changeme_dispatcher_secret_key + keep_access_key: changeme_keep_access_key + keep_secret_key: changeme_keep_secret_key + + AuditLogs: + Section_to_ignore: + - some_random_value + + ### VOLUMES + ## This should usually match all your `keepstore` instances + Volumes: + # the volume name will be composed with + # -nyw5e- + fixme-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.fixme.example.net:25107: + ReadOnly: false + Replication: 2 + Driver: Directory + DriverParameters: + Root: /tmp + + Users: + NewUsersAreActive: true + AutoAdminFirstUser: true + AutoSetupNewUsers: true + AutoSetupNewUsersWithRepository: true + + Services: + Controller: + ExternalURL: https://fixme.example.net + InternalURLs: + http://127.0.0.2:8003: {} + DispatchCloud: + InternalURLs: + http://fixme.example.net:9006: {} + Keepbalance: + InternalURLs: + http://fixme.example.net:9005: {} + Keepproxy: + ExternalURL: https://keep.fixme.example.net + InternalURLs: + http://127.0.0.2:25100: {} + Keepstore: + InternalURLs: + http://keep0.fixme.example.net:25107: {} + RailsAPI: + InternalURLs: + http://127.0.0.2:8004: {} + WebDAV: + ExternalURL: https://collections.fixme.example.net + InternalURLs: + http://127.0.0.2:9002: {} + WebDAVDownload: + ExternalURL: https://download.fixme.example.net + WebShell: + ExternalURL: https://webshell.fixme.example.net + Websocket: + ExternalURL: wss://ws.fixme.example.net/websocket + InternalURLs: + http://127.0.0.2:8005: {} + Workbench1: + ExternalURL: https://workbench.fixme.example.net + Workbench2: + ExternalURL: https://workbench2.fixme.example.net diff --git a/test/salt/pillar/arvados_dev.sls b/test/salt/pillar/arvados_dev.sls new file mode 100644 index 0000000..14450be --- /dev/null +++ b/test/salt/pillar/arvados_dev.sls @@ -0,0 +1,153 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +# The variables commented out are the default values that the formula uses. +# The uncommented values are REQUIRED values. If you don't set them, running +# this formula will fail. +arvados: + ### GENERAL CONFIG + # version: '2.0.4' + ## It makes little sense to disable this flag, but you can, if you want :) + # use_upstream_repo: true + + ## Repo URL is built with grains values. If desired, it can be completely + ## overwritten with the pillar parameter 'repo_url' + # repo: + # humanname: Arvados Official Repository + + release: development + + ## IMPORTANT!!!!! + ## api, workbench and shell require some gems, so you need to make sure ruby + ## and deps are installed in order to install and compile the gems. + ## We default to `false` in these two variables as it's expected you already + ## manage OS packages with some other tool and you don't want us messing up + ## with your setup. + ruby: + ## We set these to `true` here for testing purposes. + ## They both default to `false`. + manage_ruby: true + manage_gems_deps: true + # pkg: ruby + # gems_deps: + # - curl + # - g++ + # - gcc + # - git + # - libcurl4 + # - libcurl4-gnutls-dev + # - libpq-dev + # - libxml2 + # - libxml2-dev + # - make + # - python3-dev + # - ruby-dev + # - zlib1g-dev + + # config: + # file: /etc/arvados/config.yml + # user: root + ## IMPORTANT!!!!! + ## If you're intalling any of the rails apps (api, workbench), the group + ## should be set to that of the web server, usually `www-data` + # group: root + # mode: 640 + + ### ARVADOS CLUSTER CONFIG + cluster: + name: fixme + domain: example.net + + database: + # max concurrent connections per arvados server daemon + # connection_pool_max: 32 + name: arvados + host: 127.0.0.1 + password: changeme_arvados + user: arvados + encoding: en_US.utf8 + client_encoding: UTF8 + + tls: + # certificate: '' + # key: '' + # required to test with snakeoil certs + insecure: true + + ### TOKENS + tokens: + system_root: changeme_system_root_token + management: changeme_management_token + rails_secret: changeme_rails_secret_token + anonymous_user: changeme_anonymous_user_token + + ### KEYS + secrets: + blob_signing_key: changeme_blob_signing_key + workbench_secret_key: changeme_workbench_secret_key + dispatcher_access_key: changeme_dispatcher_access_key + dispatcher_secret_key: changeme_dispatcher_secret_key + keep_access_key: changeme_keep_access_key + keep_secret_key: changeme_keep_secret_key + + AuditLogs: + Section_to_ignore: + - some_random_value + + ### VOLUMES + ## This should usually match all your `keepstore` instances + Volumes: + # the volume name will be composed with + # -nyw5e- + fixme-nyw5e-000000000000000: + AccessViaHosts: + http://keep0.fixme.example.net:25107: + ReadOnly: false + Replication: 2 + Driver: Directory + DriverParameters: + Root: /tmp + + Users: + NewUsersAreActive: true + AutoAdminFirstUser: true + AutoSetupNewUsers: true + AutoSetupNewUsersWithRepository: true + + Services: + Controller: + ExternalURL: https://fixme.example.net + InternalURLs: + http://127.0.0.2:8003: {} + DispatchCloud: + InternalURLs: + http://fixme.example.net:9006: {} + Keepbalance: + InternalURLs: + http://fixme.example.net:9005: {} + Keepproxy: + ExternalURL: https://keep.fixme.example.net + InternalURLs: + http://127.0.0.2:25100: {} + Keepstore: + InternalURLs: + http://keep0.fixme.example.net:25107: {} + RailsAPI: + InternalURLs: + http://127.0.0.2:8004: {} + WebDAV: + ExternalURL: https://collections.fixme.example.net + InternalURLs: + http://127.0.0.2:9002: {} + WebDAVDownload: + ExternalURL: https://download.fixme.example.net + WebShell: + ExternalURL: https://webshell.fixme.example.net + Websocket: + ExternalURL: wss://ws.fixme.example.net/websocket + InternalURLs: + http://127.0.0.2:8005: {} + Workbench1: + ExternalURL: https://workbench.fixme.example.net + Workbench2: + ExternalURL: https://workbench2.fixme.example.net diff --git a/test/salt/pillar/examples/locale.sls b/test/salt/pillar/examples/locale.sls new file mode 100644 index 0000000..caa684b --- /dev/null +++ b/test/salt/pillar/examples/locale.sls @@ -0,0 +1,10 @@ +--- +locale: + present: + - "en_US.UTF-8 UTF-8" + default: + # Note: On debian systems don't write the second 'UTF-8' here or you will + # experience salt problems like: LookupError: unknown encoding: utf_8_utf_8 + # Restart the minion after you corrected this! + name: 'en_US.UTF-8' + requires: 'en_US.UTF-8 UTF-8' diff --git a/test/salt/pillar/examples/nginx_api_configuration.sls b/test/salt/pillar/examples/nginx_api_configuration.sls index c81cd59..39eb3bc 100644 --- a/test/salt/pillar/examples/nginx_api_configuration.sls +++ b/test/salt/pillar/examples/nginx_api_configuration.sls @@ -1,6 +1,4 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### ARVADOS arvados: config: @@ -16,11 +14,11 @@ nginx: overwrite: true config: - server: - - listen: '127.0.0.1:8004' + - listen: '127.0.0.2:8004' - server_name: api - root: /var/www/arvados-api/current/public - index: index.html index.htm - - access_log: {{ nginx_log }}/api.example.net-upstream.access.log combined - - error_log: {{ nginx_log }}/api.example.net-upstream.error.log + - access_log: /var/log/nginx/api.fixme.example.net-upstream.access.log combined + - error_log: /var/log/nginx/api.fixme.example.net-upstream.error.log - passenger_enabled: 'on' - client_max_body_size: 128m diff --git a/test/salt/pillar/examples/nginx_controller_configuration.sls b/test/salt/pillar/examples/nginx_controller_configuration.sls index 15cc63d..43701f3 100644 --- a/test/salt/pillar/examples/nginx_controller_configuration.sls +++ b/test/salt/pillar/examples/nginx_controller_configuration.sls @@ -1,19 +1,16 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### NGINX nginx: ### SERVER server: config: - ### STREAMS http: 'geo $external_client': default: 1 - '127.0.0.0/24': 0 + '127.0.0.0/8': 0 upstream controller_upstream: - - server: 'localhost:8003 fail_timeout=10s' + - server: '127.0.0.2:8003 fail_timeout=10s' ### SITES servers: @@ -24,7 +21,7 @@ nginx: overwrite: true config: - server: - - server_name: example.net api.example.net + - server_name: fixme.example.net - listen: - 80 default - location /.well-known: @@ -37,7 +34,7 @@ nginx: overwrite: true config: - server: - - server_name: example.net + - server_name: fixme.example.net - listen: - 443 http2 ssl - index: index.html index.htm @@ -53,6 +50,6 @@ nginx: - proxy_set_header: 'X-External-Client $external_client' # - include: 'snippets/letsencrypt.conf' - include: 'snippets/snakeoil.conf' - - access_log: {{ nginx_log }}/example.net.access.log combined - - error_log: {{ nginx_log }}/example.net.error.log + - access_log: /var/log/nginx/fixme.example.net.access.log combined + - error_log: /var/log/nginx/fixme.example.net.error.log - client_max_body_size: 128m diff --git a/test/salt/pillar/examples/nginx_keepproxy_configuration.sls b/test/salt/pillar/examples/nginx_keepproxy_configuration.sls index 3de312a..2485b37 100644 --- a/test/salt/pillar/examples/nginx_keepproxy_configuration.sls +++ b/test/salt/pillar/examples/nginx_keepproxy_configuration.sls @@ -1,6 +1,4 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### NGINX nginx: ### SERVER @@ -9,7 +7,7 @@ nginx: ### STREAMS http: upstream keepproxy_upstream: - - server: '127.0.0.1:25107 fail_timeout=10s' + - server: '127.0.0.2:25100 fail_timeout=10s' servers: managed: @@ -19,7 +17,7 @@ nginx: overwrite: true config: - server: - - server_name: keep.example.net + - server_name: keep.fixme.example.net - listen: - 80 - location /.well-known: @@ -32,7 +30,7 @@ nginx: overwrite: true config: - server: - - server_name: keep.example.net + - server_name: keep.fixme.example.net - listen: - 443 http2 ssl - index: index.html index.htm @@ -52,5 +50,5 @@ nginx: - proxy_request_buffering: 'off' # - include: 'snippets/letsencrypt.conf' - include: 'snippets/snakeoil.conf' - - access_log: {{ nginx_log }}/keepproxy.example.net.access.log combined - - error_log: {{ nginx_log }}/keepproxy.example.net.error.log + - access_log: /var/log/nginx/keepproxy.fixme.example.net.access.log combined + - error_log: /var/log/nginx/keepproxy.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_keepweb_configuration.sls b/test/salt/pillar/examples/nginx_keepweb_configuration.sls index 42a7240..cbc4aef 100644 --- a/test/salt/pillar/examples/nginx_keepweb_configuration.sls +++ b/test/salt/pillar/examples/nginx_keepweb_configuration.sls @@ -1,6 +1,4 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### NGINX nginx: ### SERVER @@ -9,7 +7,7 @@ nginx: ### STREAMS http: upstream collections_downloads_upstream: - - server: '127.0.0.1:9002 fail_timeout=10s' + - server: '127.0.0.2:9002 fail_timeout=10s' servers: managed: @@ -19,7 +17,7 @@ nginx: overwrite: true config: - server: - - server_name: collections.example.net download.example.net + - server_name: collections.fixme.example.net download.fixme.example.net - listen: - 80 - location /.well-known: @@ -33,7 +31,7 @@ nginx: overwrite: true config: - server: - - server_name: collections.example.net download.example.net + - server_name: collections.fixme.example.net download.fixme.example.net - listen: - 443 http2 ssl - index: index.html index.htm @@ -52,5 +50,5 @@ nginx: - proxy_request_buffering: 'off' # - include: 'snippets/letsencrypt.conf' - include: 'snippets/snakeoil.conf' - - access_log: {{ nginx_log }}/collections.example.net.access.log combined - - error_log: {{ nginx_log }}/collections.example.net.error.log + - access_log: /var/log/nginx/collections.fixme.example.net.access.log combined + - error_log: /var/log/nginx/collections.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_passenger.sls b/test/salt/pillar/examples/nginx_passenger.sls index 1d36e39..2c05dbd 100644 --- a/test/salt/pillar/examples/nginx_passenger.sls +++ b/test/salt/pillar/examples/nginx_passenger.sls @@ -1,9 +1,4 @@ --- -states: - - nginx.passenger - -{% set nginx_log = '/var/log/nginx' %} - ### NGINX nginx: install_from_phusionpassenger: true diff --git a/test/salt/pillar/examples/nginx_webshell_configuration.sls b/test/salt/pillar/examples/nginx_webshell_configuration.sls new file mode 100644 index 0000000..33f3dfd --- /dev/null +++ b/test/salt/pillar/examples/nginx_webshell_configuration.sls @@ -0,0 +1,71 @@ +--- +### NGINX +nginx: + ### SERVER + server: + config: + + ### STREAMS + http: + upstream webshell_upstream: + - server: '127.0.0.2:4200 fail_timeout=10s' + + ### SITES + servers: + managed: + arvados_webshell: + enabled: true + overwrite: true + config: + - server: + - server_name: webshell.fixme.example.net + - listen: + - 80 + - location /.well-known: + - root: /var/www + - location /: + - return: '301 https://$host$request_uri' + + arvados_webshell-ssl: + enabled: true + overwrite: true + config: + - server: + - server_name: webshell.fixme.example.net + - listen: + - 443 http2 ssl + - index: index.html index.htm + - location /shell.fixme.example.net: + - proxy_pass: 'http://webshell_upstream' + - proxy_read_timeout: 90 + - proxy_connect_timeout: 90 + - proxy_set_header: 'Host $http_host' + - proxy_set_header: 'X-Real-IP $remote_addr' + - proxy_set_header: X-Forwarded-Proto https + - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' + - proxy_ssl_session_reuse: 'off' + + - "if ($request_method = 'OPTIONS')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + - add_header: "'Access-Control-Max-Age' 1728000" + - add_header: "'Content-Type' 'text/plain charset=UTF-8'" + - add_header: "'Content-Length' 0" + - return: 204 + + - "if ($request_method = 'POST')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + + - "if ($request_method = 'GET')": + - add_header: "'Access-Control-Allow-Origin' '*'" + - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'" + - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'" + + # - include: 'snippets/letsencrypt.conf' + - include: 'snippets/snakeoil.conf' + - access_log: /var/log/nginx/webshell.fixme.example.net.access.log combined + - error_log: /var/log/nginx/webshell.fixme.example.net.error.log + diff --git a/test/salt/pillar/examples/nginx_websocket_configuration.sls b/test/salt/pillar/examples/nginx_websocket_configuration.sls index b6a2174..01b7788 100644 --- a/test/salt/pillar/examples/nginx_websocket_configuration.sls +++ b/test/salt/pillar/examples/nginx_websocket_configuration.sls @@ -1,6 +1,4 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### NGINX nginx: ### SERVER @@ -9,7 +7,7 @@ nginx: ### STREAMS http: upstream websocket_upstream: - - server: '127.0.0.1:8005 fail_timeout=10s' + - server: '127.0.0.2:8005 fail_timeout=10s' servers: managed: @@ -19,7 +17,7 @@ nginx: overwrite: true config: - server: - - server_name: ws.example.net + - server_name: ws.fixme.example.net - listen: - 80 - location /.well-known: @@ -32,7 +30,7 @@ nginx: overwrite: true config: - server: - - server_name: ws.example.net + - server_name: ws.fixme.example.net - listen: - 443 http2 ssl - index: index.html index.htm @@ -53,5 +51,5 @@ nginx: - proxy_request_buffering: 'off' # - include: 'snippets/letsencrypt.conf' - include: 'snippets/snakeoil.conf' - - access_log: {{ nginx_log }}/ws.example.net.access.log combined - - error_log: {{ nginx_log }}/ws.example.net.error.log + - access_log: /var/log/nginx/ws.fixme.example.net.access.log combined + - error_log: /var/log/nginx/ws.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_workbench2_configuration.sls b/test/salt/pillar/examples/nginx_workbench2_configuration.sls index 9718a96..9c2e0fa 100644 --- a/test/salt/pillar/examples/nginx_workbench2_configuration.sls +++ b/test/salt/pillar/examples/nginx_workbench2_configuration.sls @@ -1,6 +1,4 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### ARVADOS arvados: config: @@ -17,7 +15,7 @@ nginx: overwrite: true config: - server: - - server_name: workbench2.example.net + - server_name: workbench2.fixme.example.net - listen: - 80 - location /.well-known: @@ -30,7 +28,7 @@ nginx: overwrite: true config: - server: - - server_name: workbench2.example.net + - server_name: workbench2.fixme.example.net - listen: - 443 http2 ssl - index: index.html index.htm @@ -39,7 +37,9 @@ nginx: - try_files: '$uri $uri/ /index.html' - 'if (-f $document_root/maintenance.html)': - return: 503 + - location /config.json: + - return: {{ "200 '" ~ '{"API_HOST":"fixme.example.net"}' ~ "'" }} # - include: 'snippets/letsencrypt.conf' - include: 'snippets/snakeoil.conf' - - access_log: {{ nginx_log }}/workbench2.example.net.access.log combined - - error_log: {{ nginx_log }}/workbench2.example.net.error.log + - access_log: /var/log/nginx/workbench2.fixme.example.net.access.log combined + - error_log: /var/log/nginx/workbench2.fixme.example.net.error.log diff --git a/test/salt/pillar/examples/nginx_workbench_configuration.sls b/test/salt/pillar/examples/nginx_workbench_configuration.sls index 1369e00..a641bba 100644 --- a/test/salt/pillar/examples/nginx_workbench_configuration.sls +++ b/test/salt/pillar/examples/nginx_workbench_configuration.sls @@ -1,6 +1,4 @@ --- -{% set nginx_log = '/var/log/nginx' %} - ### ARVADOS arvados: config: @@ -15,7 +13,7 @@ nginx: ### STREAMS http: upstream workbench_upstream: - - server: '127.0.0.1:9000 fail_timeout=10s' + - server: '127.0.0.2:9000 fail_timeout=10s' ### SITES servers: @@ -26,7 +24,7 @@ nginx: overwrite: true config: - server: - - server_name: workbench.example.net + - server_name: workbench.fixme.example.net - listen: - 80 - location /.well-known: @@ -39,7 +37,7 @@ nginx: overwrite: true config: - server: - - server_name: workbench.example.net + - server_name: workbench.fixme.example.net - listen: - 443 http2 ssl - index: index.html index.htm @@ -54,18 +52,19 @@ nginx: - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for' # - include: 'snippets/letsencrypt.conf' - include: 'snippets/snakeoil.conf' - - access_log: {{ nginx_log }}/workbench.example.net.access.log combined - - error_log: {{ nginx_log }}/workbench.example.net.error.log + - access_log: /var/log/nginx/workbench.fixme.example.net.access.log combined + - error_log: /var/log/nginx/workbench.fixme.example.net.error.log arvados_workbench_upstream: enabled: true overwrite: true config: - server: - - listen: '127.0.0.1:9000' + - listen: '127.0.0.2:9000' - server_name: workbench - root: /var/www/arvados-workbench/current/public - index: index.html index.htm + - passenger_enabled: 'on' # yamllint disable-line rule:line-length - - access_log: {{ nginx_log }}/workbench.example.net-upstream.access.log combined - - error_log: {{ nginx_log }}/workbench.example.net-upstream.error.log + - access_log: /var/log/nginx/workbench.fixme.example.net-upstream.access.log combined + - error_log: /var/log/nginx/workbench.fixme.example.net-upstream.error.log diff --git a/test/salt/pillar/examples/postgresql.sls b/test/salt/pillar/examples/postgresql.sls index 6cce4b9..dbca5b5 100644 --- a/test/salt/pillar/examples/postgresql.sls +++ b/test/salt/pillar/examples/postgresql.sls @@ -1,7 +1,4 @@ --- -states: - - postgres - ### POSTGRESQL postgres: use_upstream_repo: false -- 2.30.2