Merge branch '18631-shell-login-sync' 2.4-release main
authorJavier Bértoli <jbertoli@curii.com>
Wed, 6 Apr 2022 13:48:22 +0000 (10:48 -0300)
committerJavier Bértoli <jbertoli@curii.com>
Wed, 6 Apr 2022 13:48:22 +0000 (10:48 -0300)
refs #18631
Arvados-DCO-1.1-Signed-off-by: Javier Bértoli <jbertoli@curii.com>

arvados/controller/init.sls
arvados/shell/init.sls
arvados/shell/resources/init.sls [moved from arvados/controller/resources/init.sls with 100% similarity]
arvados/shell/resources/virtual_machines.sls [moved from arvados/controller/resources/virtual_machines.sls with 79% similarity]
kitchen.yml
test/salt/states/examples/arvados/shell/cron/add-login-sync.sls

index 24f84d068757cf22ef637650171f00bfd303519f..02a98b8f5fdff4dc49e88e16b289461f6cc3b127 100644 (file)
@@ -5,4 +5,3 @@ include:
   - .package
   - ..config
   - .service
-  - .resources
index 858a8e6ee0723c478f5fab6ca24f8cc252a7bfc8..26e3e8668f6cdb22c440a394c7a1cf1406cb6e58 100644 (file)
@@ -5,3 +5,4 @@ include:
   - .package
   - .config
   - .service
+  - .resources
similarity index 79%
rename from arvados/controller/resources/virtual_machines.sls
rename to arvados/shell/resources/virtual_machines.sls
index 23b8496d24eb12c609f2f161054a1057bac48aa0..18e4c16d701f65107ba7dc998ea5964410debc0e 100644 (file)
@@ -7,7 +7,7 @@
 
 {#- Get the `tplroot` from `tpldir` #}
 {%- set tplroot = tpldir.split('/')[0] %}
-{%- set sls_config_file = tplroot ~ '.config.file' %}
+{#- set sls_config_file = tplroot ~ '.config.file' #}
 {%- from tplroot ~ "/map.jinja" import arvados with context %}
 {%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
 
                       else 'arv' %}
 include:
   - ..package
-  - {{ sls_config_file }}
-  - ..service
+  {# - {{ sls_config_file }} #}
+  - ..service
 
-arvados-controller-resources-virtual-machines-jq-pkg-installed:
+arvados-shell-resources-virtual-machines-jq-pkg-installed:
   pkg.installed:
     - name: jq
 
@@ -39,7 +39,7 @@ arvados-controller-resources-virtual-machines-jq-pkg-installed:
   %}
 
 # Create the virtual machine record
-arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run:
+arvados-shell-resources-virtual-machines-{{ vm }}-record-cmd-run:
   cmd.run:
     - env:
       - ARVADOS_API_TOKEN: {{ api_token }}
@@ -54,24 +54,23 @@ arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run:
           {{ cmd_query_vm_uuid }} | \
           /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}"
     - require:
-      - pkg: arvados-controller-package-install-pkg-installed
-      - cmd: arvados-controller-service-running-service-ready-cmd-run
-      - gem: arvados-controller-package-install-gem-arvados-cli-installed
+      - pkg: arvados-shell-package-install-pkg-installed
+      - gem: arvados-shell-package-install-gem-arvados-cli-installed
 
 # We need to use the UUID generated in the previous command to see if there's a
 # scoped token for it. There's no easy way to pass the value from a shellout
 # to another state, so we store it in a temp file and use that in the next
 # command. Flaky, mostly because the `unless` clause is just checking thatg
 # the file content is a token uuid :|
-arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run:
+arvados-shell-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run:
   cmd.run:
     {%- if arvados.ruby.manage_ruby and arvados.ruby.use_rvm %}
     - prepend_path: /usr/local/rvm/gems/{{ arvados.ruby.pkg }}/bin
     {%- endif %}
     - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/{{ vm }}
     - require:
-      - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run
-      - gem: arvados-controller-package-install-gem-arvados-cli-installed
+      - cmd: arvados-shell-resources-virtual-machines-{{ vm }}-record-cmd-run
+      - gem: arvados-shell-package-install-gem-arvados-cli-installed
     - unless:
       - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/{{ vm }}
 
@@ -88,7 +87,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run:
   %}
 
 # Create the VM scoped tokens
-arvados-controller-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run:
+arvados-shell-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run:
   cmd.run:
     - env:
       - ARVADOS_API_TOKEN: {{ api_token }}
@@ -102,9 +101,9 @@ arvados-controller-resources-virtual-machines-{{ vm }}-scoped-token-cmd-run:
           --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/'${VM_UUID}'/logins"]}'
     - unless: {{ cmd_query_scoped_token_url }}
     - require:
-      - pkg: arvados-controller-package-install-pkg-installed
-      - pkg: arvados-controller-resources-virtual-machines-jq-pkg-installed
-      - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run
-      - gem: arvados-controller-package-install-gem-arvados-cli-installed
+      - pkg: arvados-shell-package-install-pkg-installed
+      - pkg: arvados-shell-resources-virtual-machines-jq-pkg-installed
+      - cmd: arvados-shell-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run
+      - gem: arvados-shell-package-install-gem-arvados-cli-installed
 
 {%- endfor %}
index e7f200bb2b5f85281f233682bc1728a3dd420c66..0b3dcd6285915d214fbb9f9efcf73a22a4d42e1f 100644 (file)
@@ -57,53 +57,53 @@ platforms:
     driver:
       image: saltimages/salt-master-py3:centos-7
 
-  ## SALT `3004.0`
-  - name: debian-11-3004.0-py3
+  ## SALT `3004.1`
+  - name: debian-11-3004.1-py3
     driver:
-      image: saltimages/salt-3004.0-py3:debian-11
-  - name: debian-10-3004.0-py3
+      image: saltimages/salt-3004.1-py3:debian-11
+  - name: debian-10-3004.1-py3
     driver:
-      image: saltimages/salt-3004.0-py3:debian-10
-  - name: ubuntu-2004-3004.0-py3
+      image: saltimages/salt-3004.1-py3:debian-10
+  - name: ubuntu-2004-3004.1-py3
     driver:
-      image: saltimages/salt-3004.0-py3:ubuntu-20.04
-  - name: ubuntu-1804-3004.0-py3
+      image: saltimages/salt-3004.1-py3:ubuntu-20.04
+  - name: ubuntu-1804-3004.1-py3
     driver:
-      image: saltimages/salt-3004.0-py3:ubuntu-18.04
-  - name: centos-7-3004.0-py3
+      image: saltimages/salt-3004.1-py3:ubuntu-18.04
+  - name: centos-7-3004.1-py3
     driver:
-      image: saltimages/salt-3004.0-py3:centos-7
+      image: saltimages/salt-3004.1-py3:centos-7
 
-  ## SALT `3003.3`
-  - name: debian-11-3003.3-py3
+  ## SALT `3003.4`
+  - name: debian-11-3003.4-py3
     driver:
-      image: saltimages/salt-3003.3-py3:debian-11
-  - name: debian-10-3003.3-py3
+      image: saltimages/salt-3003.4-py3:debian-11
+  - name: debian-10-3003.4-py3
     driver:
-      image: saltimages/salt-3003.3-py3:debian-10
-  - name: ubuntu-2004-3003.3-py3
+      image: saltimages/salt-3003.4-py3:debian-10
+  - name: ubuntu-2004-3003.4-py3
     driver:
-      image: saltimages/salt-3003.3-py3:ubuntu-20.04
-  - name: ubuntu-1804-3003.3-py3
+      image: saltimages/salt-3003.4-py3:ubuntu-20.04
+  - name: ubuntu-1804-3003.4-py3
     driver:
-      image: saltimages/salt-3003.3-py3:ubuntu-18.04
-  - name: centos-7-3003.3-py3
+      image: saltimages/salt-3003.4-py3:ubuntu-18.04
+  - name: centos-7-3003.4-py3
     driver:
-      image: saltimages/salt-3003.3-py3:centos-7
+      image: saltimages/salt-3003.4-py3:centos-7
 
-  ## SALT `3002.7`
-  - name: debian-10-3002.7-py3
+  ## SALT `3002.8`
+  - name: debian-10-3002.8-py3
     driver:
-      image: saltimages/salt-3002.7-py3:debian-10
-  - name: ubuntu-2004-3002.7-py3
+      image: saltimages/salt-3002.8-py3:debian-10
+  - name: ubuntu-2004-3002.8-py3
     driver:
-      image: saltimages/salt-3002.7-py3:ubuntu-20.04
-  - name: ubuntu-1804-3002.7-py3
+      image: saltimages/salt-3002.8-py3:ubuntu-20.04
+  - name: ubuntu-1804-3002.8-py3
     driver:
-      image: saltimages/salt-3002.7-py3:ubuntu-18.04
-  - name: centos-7-3002.7-py3
+      image: saltimages/salt-3002.8-py3:ubuntu-18.04
+  - name: centos-7-3002.8-py3
     driver:
-      image: saltimages/salt-3002.7-py3:centos-7
+      image: saltimages/salt-3002.8-py3:centos-7
 
 verifier:
   # https://www.inspec.io/
@@ -148,6 +148,7 @@ suites:
             - arvados.controller
             - arvados.dispatcher
             - arvados.keepbalance
+            - arvados.shell.resources.virtual_machines
       pillars:
         top.sls:
           base:
index fdcf59982805335c3bdd3933924f7d4a70a785fc..a14808b033498715c97f592f6eacf0e4497413ca 100644 (file)
@@ -1,22 +1,23 @@
-# -*- coding: utf-8 -*-
-# vim: ft=sls
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
 
 # This state tries to query the controller using the parameters set in
 # the `arvados.cluster.resources.virtual_machines` pillar, to get the
-# scoped_token for the host and configure the arvados login-sync cron
-# as described in https://doc.arvados.org/v2.0/install/install-shell-server.html
+# ARVADOS_VIRTUAL_MACHINE_UUID for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/main/install/install-shell-server.html
 
-{#- Get the `tplroot` from `tpldir` #}
-{%- set tplroot = tpldir.split('/')[0] %}
-{%- set sls_config_file = tplroot ~ '.config.file' %}
-{%- from tplroot ~ "/map.jinja" import arvados with context %}
-{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
+{%- set curr_tpldir = tpldir %}
+{%- set tpldir = 'arvados' %}
+{%- from "arvados/map.jinja" import arvados with context %}
+{%- from "arvados/libtofs.jinja" import files_switch with context %}
+{%- set tpldir = curr_tpldir %}
 
 {%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
 {%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
 {%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
 
-examples-arvados-shell-cron-add-login-sync-add-jq-pkg-installed:
+examples_arvados_shell_cron_add_login_sync_add_jq_pkg_installed:
   pkg.installed:
     - name: jq
 
@@ -26,68 +27,60 @@ examples-arvados-shell-cron-add-login-sync-add-jq-pkg-installed:
   # Check if any of the specified virtual_machines parameters corresponds to this instance
   # It should be an error if we get more than one occurrence
   {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
-         backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
-                    grains['ipv4'] + grains['ipv6'] %}
+         vm_params.backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+                               grains['ipv4'] + grains['ipv6'] %}
 
+    # We need to query the VM UUID
     {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
                                 ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
     %}
 
-# We need to use the UUID generated in the previous command to see if there's a
-# scoped token for it. There's no easy way to pass the value from a shellout
-# to another state, so we store it in a temp file and use that in the next
-# command. Flaky, mostly because the `unless` clause is just checking thatg
-# the file content is a token uuid :|
-examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-vm_uuid-cmd-run:
+examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_get_vm_uuid_cmd_run:
   cmd.run:
     - env:
       - ARVADOS_API_TOKEN: {{ api_token }}
       - ARVADOS_API_HOST: {{ api_host }}
-    - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/{{ vm }}
-    - require:
-      - cmd: examples-arvados-shell-cron-add-login-sync-add-resources-virtual-machines-{{ vm }}-record-cmd-run
+      - ARVADOS_API_HOST_INSECURE: {{ arvados.cluster.tls.insecure | default(false) }}
+    - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/vm_uuid_{{ vm }}
     - unless:
-      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/{{ vm }}
-
-  # There's no direct way to query the scoped_token for a given virtual_machine
-  # so we need to parse the api_client_authorization list through some jq
-  {%- set cmd_query_scoped_token_url = 'VM_UUID=$(cat /tmp/' ~ vm ~ ') && ' ~
-                                       'arv api_client_authorization list | ' ~
-                                       '/usr/bin/jq -e \'.items[]| select(.scopes[] == "GET ' ~
-                                       '/arvados/v1/virtual_machines/\'${VM_UUID}\'/logins") | ' ~
-                                       '.api_token\' | head -1 | tee /tmp/sctk' ~ vm ~ ' && ' ~
-                                       'unset VM_UUID'
-  %}
-
-examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-scoped_token-cmd-run:
-  cmd.run:
-    - env:
-      - ARVADOS_API_TOKEN: {{ api_token }}
-      - ARVADOS_API_HOST: {{ api_host }}
-    - name: {{ cmd_query_scoped_token_url }}
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
     - require:
-      - cmd: examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-vm_uuid-cmd-run
-    - unless:
-      - test -s /tmp/sctk{{ vm }}
+      - gem: arvados-shell-package-install-gem-arvados-cli-installed
 
-examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-host-cron-env-present:
+examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_cron_env_present:
   cron.env_present:
     - name: ARVADOS_API_HOST
     - value: {{ api_host }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
 
-examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-api-cron-token-env-present:
+examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_token_cron_env_present:
   cron.env_present:
     - name: ARVADOS_API_TOKEN
-    - value: __slot__:salt:cmd.run(cat /tmp/sctk{{ vm }})
+    - value: {{ api_token }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
+
+examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_api_host_insecure_cron_env_present:
+  cron.env_present:
+    - name: ARVADOS_API_HOST_INSECURE
+    - value: {{ arvados.cluster.tls.insecure | default(false) }}
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
 
-examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-api-cron-token-env-present:
+examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_virtual_machine_uuid_cron_env_present:
   cron.env_present:
     - name: ARVADOS_VIRTUAL_MACHINE_UUID
-    - value: __slot__:salt:cmd.run(cat /tmp/{{ vm }})
+    - value: __slot__:salt:cmd.run("cat /tmp/vm_uuid_{{ vm }}")
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
 
-examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-login-sync-cron-present:
+examples_arvados_shell_cron_add_login_sync_add_{{ vm }}_arvados_login_sync_cron_present:
   cron.present:
-    minute: '*/2'
-    cmd: arvados-login-sync
+    - name: /usr/local/bin/arvados-login-sync
+    - minute: '*/2'
+    - onlyif:
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/vm_uuid_{{ vm }}
 
+  {%- endif %}
 {%- endfor %}