docs(examples): add login-sync example for orchestrating crons

diff --git a/arvados/controller/resources/virtual_machines.sls b/arvados/controller/resources/virtual_machines.sls
index 7303dac1c1874eec279548fdfbb968bd03ee98c8..3ee67f38d1bec18e7ff6baa374c7c97e5916b0a6 100644 (file)
--- a/arvados/controller/resources/virtual_machines.sls
+++ b/arvados/controller/resources/virtual_machines.sls
@@ -42,7 +42,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run:
           --virtual-machine '{"hostname":"{{ vm_name }}" }'
     - unless: |
           {{ cmd_query_vm_uuid }} | \
           --virtual-machine '{"hostname":"{{ vm_name }}" }'
     - unless: |
           {{ cmd_query_vm_uuid }} | \

-          /bin/grep -qE "fixme-2x53u-[a-z0-9]{15}"
+          /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}"

     - require:
       - pkg: arvados-controller-package-install-pkg-installed
       - cmd: arvados-controller-service-running-service-ready-cmd-run
     - require:
       - pkg: arvados-controller-package-install-pkg-installed
       - cmd: arvados-controller-service-running-service-ready-cmd-run


@@ -58,7 +58,7 @@ arvados-controller-resources-virtual-machines-{{ vm }}-get-vm_uuid-cmd-run:
     - require:
       - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run
     - unless:
     - require:
       - cmd: arvados-controller-resources-virtual-machines-{{ vm }}-record-cmd-run
     - unless:

-      - /bin/grep -qE "fixme-2x53u-[a-z0-9]{15}" /tmp/{{ vm }}
+      - /bin/grep -qE "[a-z0-9]{5}-2x53u-[a-z0-9]{15}" /tmp/{{ vm }}

 
   # There's no direct way to query the scoped_token for a given virtual_machine
   # so we need to parse the api_client_authorization list through some jq
 
   # There's no direct way to query the scoped_token for a given virtual_machine
   # so we need to parse the api_client_authorization list through some jq

diff --git a/arvados/controller/service/running.sls b/arvados/controller/service/running.sls
index 6c95b3c9f1e17928242164d1412c1e5782332065..7f6af5136609ae3b63ed0f4df78927496df63fce 100644 (file)
--- a/arvados/controller/service/running.sls
+++ b/arvados/controller/service/running.sls
@@ -30,7 +30,7 @@ arvados-controller-service-running-service-ready-cmd-run:
           echo 'waiting for API to be ready...'
           sleep 1
         done
           echo 'waiting for API to be ready...'
           sleep 1
         done

-    - timeout: 520
+    - timeout: 120

     - unless: |
         curl -s {{  arvados.cluster.Services.Controller.ExternalURL }} | \
         grep -qE "req-[a-z0-9]{20}.{5}error_token"
     - unless: |
         curl -s {{  arvados.cluster.Services.Controller.ExternalURL }} | \
         grep -qE "req-[a-z0-9]{20}.{5}error_token"

diff --git a/test/salt/states/examples/arvados/README.md b/test/salt/states/examples/arvados/README.md
new file mode 100644 (file)
index 0000000..07b01d5
--- /dev/null
+++ b/test/salt/states/examples/arvados/README.md
@@ -0,0 +1,3 @@
+These state are examples that might help you automate some Arvados' configuration
+tasks. As they require orchestration (ie, API/controller running before being used)
+they're not included in the formula, as this orchestration is outside of its scope.

diff --git a/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls b/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls
new file mode 100644 (file)
index 0000000..353ac6a
--- /dev/null
+++ b/test/salt/states/examples/arvados/shell/cron/add-login-sync.sls
@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+# This state tries to query the controller using the parameters set in
+# the `arvados.cluster.resources.virtual_machines` pillar, to get the
+# scoped_token for the host and configure the arvados login-sync cron
+# as described in https://doc.arvados.org/v2.0/install/install-shell-server.html
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- set sls_config_file = tplroot ~ '.config.file' %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+examples-arvados-shell-cron-add-login-sync-add-jq-pkg-installed:
+  pkg.installed:
+    - name: jq
+
+{%- for vm, vm_params in virtual_machines.items() %}
+  {%- set vm_name = vm_params.name | default(vm) %}
+
+  # Check if any of the specified virtual_machines parameters corresponds to this instance
+  # It should be an error if we get more than one occurrence
+  {%- if vm_name in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] or
+         backend in [grains['id'], grains['host'], grains['fqdn'], grains['nodename']] +
+                    grains['ipv4'] + grains['ipv6'] %}
+
+    {%- set cmd_query_vm_uuid = 'arv --short virtual_machine list' ~
+                                ' --filters \'[["hostname", "=", "' ~ vm_name ~ '"]]\''
+    %}
+
+# We need to use the UUID generated in the previous command to see if there's a
+# scoped token for it. There's no easy way to pass the value from a shellout
+# to another state, so we store it in a temp file and use that in the next
+# command. Flaky, mostly because the `unless` clause is just checking thatg
+# the file content is a token uuid :|
+examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-vm_uuid-cmd-run:
+  cmd.run:
+    - env:
+      - ARVADOS_API_TOKEN: {{ api_token }}
+      - ARVADOS_API_HOST: {{ api_host }}
+    - name: {{ cmd_query_vm_uuid }} | head -1 | tee /tmp/{{ vm }}
+    - require:
+      - cmd: examples-arvados-shell-cron-add-login-sync-add-resources-virtual-machines-{{ vm }}-record-cmd-run
+    - unless:
+      - /bin/grep -qE "fixme-2x53u-[a-z0-9]{15}" /tmp/{{ vm }}
+
+  # There's no direct way to query the scoped_token for a given virtual_machine
+  # so we need to parse the api_client_authorization list through some jq
+  {%- set cmd_query_scoped_token_url = 'VM_UUID=$(cat /tmp/' ~ vm ~ ') && ' ~
+                                       'arv api_client_authorization list | ' ~
+                                       '/usr/bin/jq -e \'.items[]| select(.scopes[] == "GET ' ~
+                                       '/arvados/v1/virtual_machines/\'${VM_UUID}\'/logins") | ' ~
+                                       '.api_token\' | head -1 | tee /tmp/sctk' ~ vm ~ ' && ' ~
+                                       'unset VM_UUID'
+  %}
+
+examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-scoped_token-cmd-run:
+  cmd.run:
+    - env:
+      - ARVADOS_API_TOKEN: {{ api_token }}
+      - ARVADOS_API_HOST: {{ api_host }}
+    - name: {{ cmd_query_scoped_token_url }}
+    - require:
+      - cmd: examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-get-vm_uuid-cmd-run
+    - unless:
+      - test -s /tmp/sctk-{{ vm }}
+
+examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-host-cron-env-present:
+  cron.env_present:
+    - name: ARVADOS_API_HOST
+    - value: {{ api_host }}
+
+examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-api-cron-token-env-present:
+  cron.env_present:
+    - name: ARVADOS_API_TOKEN
+    - value: __slot__:salt:cmd.run(cat /tmp/sctk{{ vm }})
+
+examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-api-cron-token-env-present:
+  cron.env_present:
+    - name: ARVADOS_VIRTUAL_MACHINE_UUID
+    - value: __slot__:salt:cmd.run(cat /tmp/{{ vm }})
+
+examples-arvados-shell-cron-add-login-sync-add-{{ vm }}-arvados-login-sync-cron-present:
+  cron.present:
+    minute: '*/2'
+    cmd: arvados-login-sync
+
+{%- endfor %}