From: Javier Bértoli Date: Wed, 6 Apr 2022 13:48:22 +0000 (-0300) Subject: Merge branch '18631-shell-login-sync' X-Git-Url: https://git.arvados.org/arvados-formula.git/commitdiff_plain/HEAD?hp=0bd9d5075493225a0685a41e8cef35c3acbee0bf Merge branch '18631-shell-login-sync' refs #18631 Arvados-DCO-1.1-Signed-off-by: Javier Bértoli --- diff --git a/arvados/controller/init.sls b/arvados/controller/init.sls index 24f84d0..02a98b8 100644 --- a/arvados/controller/init.sls +++ b/arvados/controller/init.sls @@ -5,4 +5,3 @@ include: - .package - ..config - .service - - .resources diff --git a/arvados/shell/init.sls b/arvados/shell/init.sls index 858a8e6..26e3e86 100644 --- a/arvados/shell/init.sls +++ b/arvados/shell/init.sls @@ -5,3 +5,4 @@ include: - .package - .config - .service + - .resources diff --git a/arvados/controller/resources/init.sls b/arvados/shell/resources/init.sls similarity index 100% rename from arvados/controller/resources/init.sls rename to arvados/shell/resources/init.sls diff --git a/arvados/controller/resources/virtual_machines.sls b/arvados/shell/resources/virtual_machines.sls similarity index 79% rename from arvados/controller/resources/virtual_machines.sls rename to arvados/shell/resources/virtual_machines.sls index 23b8496..18e4c16 100644 --- a/arvados/controller/resources/virtual_machines.sls +++ b/arvados/shell/resources/virtual_machines.sls @@ -7,7 +7,7 @@ {#- Get the `tplroot` from `tpldir` #} {%- set tplroot = tpldir.split('/')[0] %} -{%- set sls_config_file = tplroot ~ '.config.file' %} +{#- set sls_config_file = tplroot ~ '.config.file' #} {%- from tplroot ~ "/map.jinja" import arvados with context %} {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} @@ -20,10 +20,10 @@ else 'arv' %} include: - ..package - - {{ sls_config_file }} - - ..service + {# - {{ sls_config_file }} #} + # - ..service -arvados-controller-resources-virtual-machines-jq-pkg-installed: +arvados-shell-resources-virtual-machines-jq-pkg-installed: pkg.installed: - name: jq @@ -39,7 +39,7 @@ arvados-controller-resources-virtual-machines-jq-pkg-installed: %} # Create the virtual machine record -arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run: +arvados-shell-resources-virtual-machines-{{ vm }}-record-cmd-run: cmd.run: - env: - ARVADOS_API_TOKEN: {{ api_token }} @@ -54,24 +54,23 @@ arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run: {{ cmd_query_vm_uuid }} | \ /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" - require: - - pkg: arvados-controller-package-install-pkg-installed - - cmd: arvados-controller-service-running-service-ready-cmd-run - - gem: arvados-controller-package-install-gem-arvados-cli-installed + - pkg: arvados-shell-package-install-pkg-installed + - gem: arvados-shell-package-install-gem-arvados-cli-installed # We need to use the UUID generated in the previous command to see if there's a # scoped token for it. There's no easy way to pass the value from a shellout # to another state, so we store it in a temp file and use that in the next # command. Flaky, mostly because the `unless` clause is just checking thatg # the file content is a token uuid :| -arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run: +arvados-shell-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run: cmd.run: {%- if arvados.ruby.manage_ruby and arvados.ruby.use_rvm %} - prepend_path: /usr/local/rvm/gems/{{ arvados.ruby.pkg }}/bin {%- endif %} - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/{{ vm }} - require: - - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run - - gem: arvados-controller-package-install-gem-arvados-cli-installed + - cmd: arvados-shell-resources-virtual-machines-{{ vm }}-record-cmd-run + - gem: arvados-shell-package-install-gem-arvados-cli-installed - unless: - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/{{ vm }} @@ -88,7 +87,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run: %} # Create the VM scoped tokens -arvados-controller-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run: +arvados-shell-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run: cmd.run: - env: - ARVADOS_API_TOKEN: {{ api_token }} @@ -102,9 +101,9 @@ arvados-controller-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run: --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/'${VM_UUID}'/logins"]}' - unless: {{ cmd_query_scoped_token_url }} - require: - - pkg: arvados-controller-package-install-pkg-installed - - pkg: arvados-controller-resources-virtual-machines-jq-pkg-installed - - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run - - gem: arvados-controller-package-install-gem-arvados-cli-installed + - pkg: arvados-shell-package-install-pkg-installed + - pkg: arvados-shell-resources-virtual-machines-jq-pkg-installed + - cmd: arvados-shell-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run + - gem: arvados-shell-package-install-gem-arvados-cli-installed {%- endfor %} diff --git a/kitchen.yml b/kitchen.yml index e7f200b..0b3dcd6 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -57,53 +57,53 @@ platforms: driver: image: saltimages/salt-master-py3:centos-7 - ## SALT `3004.0` - - name: debian-11-3004.0-py3 + ## SALT `3004.1` + - name: debian-11-3004.1-py3 driver: - image: saltimages/salt-3004.0-py3:debian-11 - - name: debian-10-3004.0-py3 + image: saltimages/salt-3004.1-py3:debian-11 + - name: debian-10-3004.1-py3 driver: - image: saltimages/salt-3004.0-py3:debian-10 - - name: ubuntu-2004-3004.0-py3 + image: saltimages/salt-3004.1-py3:debian-10 + - name: ubuntu-2004-3004.1-py3 driver: - image: saltimages/salt-3004.0-py3:ubuntu-20.04 - - name: ubuntu-1804-3004.0-py3 + image: saltimages/salt-3004.1-py3:ubuntu-20.04 + - name: ubuntu-1804-3004.1-py3 driver: - image: saltimages/salt-3004.0-py3:ubuntu-18.04 - - name: centos-7-3004.0-py3 + image: saltimages/salt-3004.1-py3:ubuntu-18.04 + - name: centos-7-3004.1-py3 driver: - image: saltimages/salt-3004.0-py3:centos-7 + image: saltimages/salt-3004.1-py3:centos-7 - ## SALT `3003.3` - - name: debian-11-3003.3-py3 + ## SALT `3003.4` + - name: debian-11-3003.4-py3 driver: - image: saltimages/salt-3003.3-py3:debian-11 - - name: debian-10-3003.3-py3 + image: saltimages/salt-3003.4-py3:debian-11 + - name: debian-10-3003.4-py3 driver: - image: saltimages/salt-3003.3-py3:debian-10 - - name: ubuntu-2004-3003.3-py3 + image: saltimages/salt-3003.4-py3:debian-10 + - name: ubuntu-2004-3003.4-py3 driver: - image: saltimages/salt-3003.3-py3:ubuntu-20.04 - - name: ubuntu-1804-3003.3-py3 + image: saltimages/salt-3003.4-py3:ubuntu-20.04 + - name: ubuntu-1804-3003.4-py3 driver: - image: saltimages/salt-3003.3-py3:ubuntu-18.04 - - name: centos-7-3003.3-py3 + image: saltimages/salt-3003.4-py3:ubuntu-18.04 + - name: centos-7-3003.4-py3 driver: - image: saltimages/salt-3003.3-py3:centos-7 + image: saltimages/salt-3003.4-py3:centos-7 - ## SALT `3002.7` - - name: debian-10-3002.7-py3 + ## SALT `3002.8` + - name: debian-10-3002.8-py3 driver: - image: saltimages/salt-3002.7-py3:debian-10 - - name: ubuntu-2004-3002.7-py3 + image: saltimages/salt-3002.8-py3:debian-10 + - name: ubuntu-2004-3002.8-py3 driver: - image: saltimages/salt-3002.7-py3:ubuntu-20.04 - - name: ubuntu-1804-3002.7-py3 + image: saltimages/salt-3002.8-py3:ubuntu-20.04 + - name: ubuntu-1804-3002.8-py3 driver: - image: saltimages/salt-3002.7-py3:ubuntu-18.04 - - name: centos-7-3002.7-py3 + image: saltimages/salt-3002.8-py3:ubuntu-18.04 + - name: centos-7-3002.8-py3 driver: - image: saltimages/salt-3002.7-py3:centos-7 + image: saltimages/salt-3002.8-py3:centos-7 verifier: # https://www.inspec.io/ @@ -148,6 +148,7 @@ suites: - arvados.controller - arvados.dispatcher - arvados.keepbalance + - arvados.shell.resources.virtual_machines pillars: top.sls: base: diff --git a/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls b/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls index fdcf599..a14808b 100644 --- a/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls +++ b/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls @@ -1,22 +1,23 @@ -# -*- coding: utf-8 -*- -# vim: ft=sls +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 # This state tries to query the controller using the parameters set in # the `arvados.cluster.resources.virtual_machines` pillar, to get the -# scoped_token for the host and configure the arvados login-sync cron -# as described in https://doc.arvados.org/v2.0/install/install-shell-server.html +# ARVADOS_VIRTUAL_MACHINE_UUID for the host and configure the arvados login-sync cron +# as described in https://doc.arvados.org/main/install/install-shell-server.html -{#- Get the `tplroot` from `tpldir` #} -{%- set tplroot = tpldir.split('/')[0] %} -{%- set sls_config_file = tplroot ~ '.config.file' %} -{%- from tplroot ~ "/map.jinja" import arvados with context %} -{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %} +{%- set curr_tpldir = tpldir %} +{%- set tpldir = 'arvados' %} +{%- from "arvados/map.jinja" import arvados with context %} +{%- from "arvados/libtofs.jinja" import files_switch with context %} +{%- set tpldir = curr_tpldir %} {%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %} {%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %} {%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %} -examples-arvados-shell-cron-add-login-sync-add-jq-pkg-installed: +examples_arvados_shell_cron_add_login_sync_add_jq_pkg_installed: pkg.installed: - name: jq @@ -26,68 +27,60 @@ examples-arvados-shell-cron-add-login-sync-add-jq-pkg-installed: # Check if any of the specified virtual_machines parameters corresponds to this instance # It should be an error if we get more than one occurrence {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or - backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] + - grains['ipv4'] + grains['ipv6'] %} + vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] + + grains['ipv4'] + grains['ipv6'] %} + # We need to query the VM UUID {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~ ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\'' %} -# We need to use the UUID generated in the previous command to see if there's a -# scoped token for it. There's no easy way to pass the value from a shellout -# to another state, so we store it in a temp file and use that in the next -# command. Flaky, mostly because the `unless` clause is just checking thatg -# the file content is a token uuid :| -examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-vm_uuid-cmd-run: +examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run: cmd.run: - env: - ARVADOS_API_TOKEN: {{ api_token }} - ARVADOS_API_HOST: {{ api_host }} - - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/{{ vm }} - - require: - - cmd: examples-arvados-shell-cron-add-login-sync-add-resources-virtual-machines-{{ vm }}-record-cmd-run + - ARVADOS_API_HOST_INSECURE: {{ arvados.cluster.tls.insecure | default(false) }} + - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }} - unless: - - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/{{ vm }} - - # There's no direct way to query the scoped_token for a given virtual_machine - # so we need to parse the api_client_authorization list through some jq - {%- set cmd_query_scoped_token_url = 'VM_UUID=$(cat /tmp/' ~ vm ~ ') && ' ~ - 'arv api_client_authorization list | ' ~ - '/usr/bin/jq -e \'.items[]| select(.scopes[] == "GET ' ~ - '/arvados/v1/virtual_machines/\'${VM_UUID}\'/logins") | ' ~ - '.api_token\' | head -1 | tee /tmp/sctk' ~ vm ~ ' && ' ~ - 'unset VM_UUID' - %} - -examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-scoped_token-cmd-run: - cmd.run: - - env: - - ARVADOS_API_TOKEN: {{ api_token }} - - ARVADOS_API_HOST: {{ api_host }} - - name: {{ cmd_query_scoped_token_url }} + - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }} - require: - - cmd: examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-vm_uuid-cmd-run - - unless: - - test -s /tmp/sctk{{ vm }} + - gem: arvados-shell-package-install-gem-arvados-cli-installed -examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-host-cron-env-present: +examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present: cron.env_present: - name: ARVADOS_API_HOST - value: {{ api_host }} + - onlyif: + - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }} -examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-api-cron-token-env-present: +examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present: cron.env_present: - name: ARVADOS_API_TOKEN - - value: __slot__:salt:cmd.run(cat /tmp/sctk{{ vm }}) + - value: {{ api_token }} + - onlyif: + - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }} + +examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_insecure_cron_env_present: + cron.env_present: + - name: ARVADOS_API_HOST_INSECURE + - value: {{ arvados.cluster.tls.insecure | default(false) }} + - onlyif: + - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }} -examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-api-cron-token-env-present: +examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present: cron.env_present: - name: ARVADOS_VIRTUAL_MACHINE_UUID - - value: __slot__:salt:cmd.run(cat /tmp/{{ vm }}) + - value: __slot__:salt:cmd.run("cat /tmp/vm_uuid_{{ vm }}") + - onlyif: + - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }} -examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-login-sync-cron-present: +examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present: cron.present: - minute: '*/2' - cmd: arvados-login-sync + - name: /usr/local/bin/arvados-login-sync + - minute: '*/2' + - onlyif: + - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }} + {%- endif %} {%- endfor %}