From: Javier Bértoli Date: Sat, 17 Oct 2020 00:03:41 +0000 (-0300) Subject: fix(dispatcher): add missing files for local dispatcher X-Git-Url: https://git.arvados.org/arvados-formula.git/commitdiff_plain/eb73d564b0b36810c56a39bbb2e75267521bfe5c fix(dispatcher): add missing files for local dispatcher --- diff --git a/arvados/defaults.yaml b/arvados/defaults.yaml index 38aa764..31f805e 100644 --- a/arvados/defaults.yaml +++ b/arvados/defaults.yaml @@ -78,7 +78,7 @@ arvados: # arvados-dispatch-cloud # crunch-dispatch-slurm service: - name: '' + name: crunch-dispatch-local port: '' ##### KEEPPROXY keepproxy: diff --git a/arvados/dispatcher/service/file.sls b/arvados/dispatcher/service/file.sls new file mode 100644 index 0000000..df75286 --- /dev/null +++ b/arvados/dispatcher/service/file.sls @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# vim: ft=sls + +{#- Get the `tplroot` from `tpldir` #} +{%- set tplroot = tpldir.split('/')[0] %} +{%- set sls_config_file = tplroot ~ '.config.file' %} +{%- from tplroot ~ "/map.jinja" import arvados with context %} +{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} + +include: + - ..package + - {{ sls_config_file }} + - .running + +{%- if arvados.dispatcher.pkg.name == 'crunch-dispatch-local' %} +arvados-dispatcher-service-file-file-managed-crunch-run-sh: + file.managed: + - name: /usr/local/bin/crunch-run.sh + - source: {{ files_switch(['crunch-run-sh.tmpl'], + lookup='arvados-dispatcher-service-file-file-managed-crunch-run-sh', + use_subpath=True + ) + }} + - mode: '0755' + - user: root + - group: root + - makedirs: True + - template: jinja + - context: + arvados: {{ arvados | json }} + - require: + - pkg: arvados-dispatcher-package-install-pkg-installed + +arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-service: + file.managed: + - name: /etc/systemd/system/crunch-dispatch-local.service + - source: {{ files_switch(['crunch-dispatch-local-service.tmpl'], + lookup='arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-service', + use_subpath=True + ) + }} + - mode: '0644' + - user: root + - group: root + - makedirs: True + - template: jinja + - context: + arvados: {{ arvados | json }} + - require: + - file: arvados-dispatcher-service-file-file-managed-crunch-run-sh + - pkg: arvados-dispatcher-package-install-pkg-installed + cmd.run: + - name: systemctl daemon-reload + - require_in: + - service: arvados-dispatcher-service-running-service-running + - require: + - file: arvados-dispatcher-service-file-file-managed-crunch-dispatch-local-service +{%- endif %} diff --git a/arvados/dispatcher/service/files/default/crunch-dispatch-local-service.tmpl b/arvados/dispatcher/service/files/default/crunch-dispatch-local-service.tmpl new file mode 100644 index 0000000..f0cda3e --- /dev/null +++ b/arvados/dispatcher/service/files/default/crunch-dispatch-local-service.tmpl @@ -0,0 +1,30 @@ +######################################################################## +# File managed by Salt at <{{ source }}>. +# Your changes will be overwritten. +######################################################################## +[Unit] +Description=Arvados Crunch Dispatcher for LOCAL service +Documentation=https://doc.arvados.org/ +After=network.target + +# systemd==229 (ubuntu:xenial) obeys StartLimitInterval in the [Unit] section +StartLimitInterval=0 + +# systemd>=230 (debian:9) obeys StartLimitIntervalSec in the [Unit] section +StartLimitIntervalSec=0 + +[Service] +Type=simple +EnvironmentFile=-/etc/arvados/environment +ExecStart=/usr/bin/crunch-dispatch-local -poll-interval=1 -crunch-run-command=/usr/local/bin/crunch-run.sh +# Set a reasonable default for the open file limit +LimitNOFILE=65536 +Restart=always +RestartSec=1 +LimitNOFILE=1000000 + +# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section +StartLimitInterval=0 + +[Install] +WantedBy=multi-user.target diff --git a/arvados/dispatcher/service/files/default/crunch-run-sh.tmpl b/arvados/dispatcher/service/files/default/crunch-run-sh.tmpl new file mode 100644 index 0000000..5c15293 --- /dev/null +++ b/arvados/dispatcher/service/files/default/crunch-run-sh.tmpl @@ -0,0 +1,6 @@ +######################################################################## +# File managed by Salt at <{{ source }}>. +# Your changes will be overwritten. +######################################################################## +#!/bin/sh +exec /usr/bin/crunch-run -container-enable-networking=default -container-network-mode=host $@ diff --git a/arvados/dispatcher/service/init.sls b/arvados/dispatcher/service/init.sls index 6fe4d1a..e913c0a 100644 --- a/arvados/dispatcher/service/init.sls +++ b/arvados/dispatcher/service/init.sls @@ -2,4 +2,5 @@ # vim: ft=sls include: + - .file - .running diff --git a/arvados/dispatcher/service/running.sls b/arvados/dispatcher/service/running.sls index cda4e1a..bed21c6 100644 --- a/arvados/dispatcher/service/running.sls +++ b/arvados/dispatcher/service/running.sls @@ -5,10 +5,11 @@ {%- set tplroot = tpldir.split('/')[0] %} {%- set sls_config_file = tplroot ~ '.config.file' %} {%- from tplroot ~ "/map.jinja" import arvados with context %} +{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} -{%- if arvados.dispatcher.pkg.name != 'crunch-dispatch-local' %} include: - ..package + - .file - {{ sls_config_file }} arvados-dispatcher-service-running-service-running: @@ -19,5 +20,3 @@ arvados-dispatcher-service-running-service-running: - sls: {{ sls_config_file }} - require: - pkg: arvados-dispatcher-package-install-pkg-installed - - only_if: test "{{ arvados.dispatcher.pkg.name }}" != "crunch-dispatch-local" -{%- endif %} diff --git a/test/integration/dispatcher/README.md b/test/integration/dispatcher/README.md new file mode 100644 index 0000000..4c295db --- /dev/null +++ b/test/integration/dispatcher/README.md @@ -0,0 +1,50 @@ +# InSpec Profile: `dispatcher` + +This shows the implementation of the `dispatcher` InSpec [profile](https://github.com/inspec/inspec/blob/master/docs/profiles.md). + +## Verify a profile + +InSpec ships with built-in features to verify a profile structure. + +```bash +$ inspec check dispatcher +Summary +------- +Location: dispatcher +Profile: profile +Controls: 4 +Timestamp: 2019-06-24T23:09:01+00:00 +Valid: true + +Errors +------ + +Warnings +-------- +``` + +## Execute a profile + +To run all **supported** controls on a local machine use `inspec exec /path/to/profile`. + +```bash +$ inspec exec dispatcher +.. + +Finished in 0.0025 seconds (files took 0.12449 seconds to load) +8 examples, 0 failures +``` + +## Execute a specific control from a profile + +To run one control from the profile use `inspec exec /path/to/profile --controls name`. + +```bash +$ inspec exec dispatcher --controls package +. + +Finished in 0.0025 seconds (files took 0.12449 seconds to load) +1 examples, 0 failures +``` + +See an [example control here](https://github.com/inspec/inspec/blob/master/examples/profile/controls/example.rb). diff --git a/test/integration/dispatcher/controls/config_spec.rb b/test/integration/dispatcher/controls/config_spec.rb new file mode 100644 index 0000000..bbc54f8 --- /dev/null +++ b/test/integration/dispatcher/controls/config_spec.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +dispatcher_stanza = <<-DISPATCHER_STANZA + DispatchCloud: + InternalURLs: + http://fixme.example.net:9006: {} +DISPATCHER_STANZA + +control 'arvados configuration' do + title 'should match desired dispatcher lines' + + describe file('/etc/arvados/config.yml') do + it { should be_file } + it { should be_owned_by 'root' } + # We're testing it in the API instance, so group will be nginx's + it { should be_grouped_into 'www-data' } + its('mode') { should cmp '0640' } + its('content') do + should include( + 'File managed by Salt at .' + ) + end + its('content') { should include(dispatcher_stanza) } + end +end diff --git a/test/integration/dispatcher/controls/files_spec.rb b/test/integration/dispatcher/controls/files_spec.rb new file mode 100644 index 0000000..17945a0 --- /dev/null +++ b/test/integration/dispatcher/controls/files_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +control 'arvados dispatcher service' do + impact 0.5 + title 'files should exist' + + describe file('/usr/local/bin/crunch-run.sh') do + it { should be_file } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } + its('mode') { should cmp '0755' } + its('content') do + should include( + # rubocop:disable Metrics/LineLength + 'File managed by Salt at .' + # rubocop:enable Metrics/LineLength + ) + end + end + describe file('/etc/systemd/system/crunch-dispatch-local.service') do + it { should be_file } + it { should be_owned_by 'root' } + it { should be_grouped_into 'root' } + its('mode') { should cmp '0644' } + its('content') do + should include( + # rubocop:disable Metrics/LineLength + 'File managed by Salt at .' + # rubocop:enable Metrics/LineLength + ) + end + end +end diff --git a/test/integration/dispatcher/controls/packages_spec.rb b/test/integration/dispatcher/controls/packages_spec.rb new file mode 100644 index 0000000..03f3e6b --- /dev/null +++ b/test/integration/dispatcher/controls/packages_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +control 'arvados dispatcher package' do + title 'should be installed' + + describe package('crunch-dispatch-local') do + it { should be_installed } + end +end diff --git a/test/integration/dispatcher/controls/services_spec.rb b/test/integration/dispatcher/controls/services_spec.rb new file mode 100644 index 0000000..465746e --- /dev/null +++ b/test/integration/dispatcher/controls/services_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +control 'arvados dispatcher service' do + impact 0.5 + title 'should be running and enabled' + + describe service('crunch-dispatch-local') do + it { should be_enabled } + it { should be_running } + end +end diff --git a/test/integration/dispatcher/inspec.yml b/test/integration/dispatcher/inspec.yml new file mode 100644 index 0000000..4a0fcb1 --- /dev/null +++ b/test/integration/dispatcher/inspec.yml @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +# vim: ft=yaml +--- +name: dispatcher +title: arvados formula dispatcher submodule +maintainer: SaltStack Formulas +license: Apache-2.0 +summary: Verify that the arvados formula dispatcher is setup and configured correctly +supports: + - platform-name: debian + - platform-name: ubuntu + - platform-name: centos diff --git a/test/integration/shell/controls/config_spec.rb b/test/integration/shell/controls/config_spec.rb index 9fa65b7..13bfa24 100644 --- a/test/integration/shell/controls/config_spec.rb +++ b/test/integration/shell/controls/config_spec.rb @@ -20,9 +20,9 @@ control 'shellinabox configuration' do its('mode') { should cmp '0644' } its('content') do should include( - # rubocop:disable Metrics/LineLength + # rubocop:disable Layout/LineLength 'File managed by Salt at .' - # rubocop:enable Metrics/LineLength + # rubocop:enable Layout/LineLength ) end its('content') { should include(siab_stanza) } @@ -39,9 +39,9 @@ control 'libpam-arvados configuration' do its('mode') { should cmp '0644' } its('content') do should include( - # rubocop:disable Metrics/LineLength + # rubocop:disable Layout/LineLength 'File managed by Salt at .' - # rubocop:enable Metrics/LineLength + # rubocop:enable Layout/LineLength ) end its('content') { should include(libpam_stanza) }