feat(resources): add state to register virtual_machines

diff --git a/arvados/api/init.sls b/arvados/api/init.sls
index 02a98b8f5fdff4dc49e88e16b289461f6cc3b127..24f84d068757cf22ef637650171f00bfd303519f 100644 (file)
--- a/arvados/api/init.sls
+++ b/arvados/api/init.sls
@@ -5,3 +5,4 @@ include:
   - .package
   - ..config
   - .service
   - .package
   - ..config
   - .service

+  - .resources

diff --git a/arvados/api/resources/init.sls b/arvados/api/resources/init.sls
new file mode 100644 (file)
index 0000000..f3f78a3
--- /dev/null
+++ b/arvados/api/resources/init.sls
@@ -0,0 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+include:
+  - .virtual_machines

diff --git a/arvados/api/resources/virtual_machines.sls b/arvados/api/resources/virtual_machines.sls
new file mode 100644 (file)
index 0000000..a2af30a
--- /dev/null
+++ b/arvados/api/resources/virtual_machines.sls
@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# vim: ft=sls
+
+{#- Get the `tplroot` from `tpldir` #}
+{%- set tplroot = tpldir.split('/')[0] %}
+{%- set sls_config_file = tplroot ~ '.config.file' %}
+{%- from tplroot ~ "/map.jinja" import arvados with context %}
+{%- from tplroot ~ "/libtofs.jinja" import files_switch with context %}
+
+{%- set virtual_machines = arvados.cluster.resources.virtual_machines | default({}) %}
+{%- set api_token = arvados.cluster.tokens.system_root | yaml_encode %}
+{%- set api_host = arvados.cluster.Services.Controller.ExternalURL | regex_replace('^http(s?)://', '', ignorecase=true) %}
+
+include:
+  - ..package
+  - {{ sls_config_file }}
+  - ..service
+
+{%- for vm, vm_params in virtual_machines.items() %}
+  {%- set vm_name = vm_params.name | default(vm) %}
+  {%- set vm_backend = vm_params.backend | default(vm_name) %}
+  {%- set vm_port = vm_params.port | default(4200) %}
+
+arvados-api-resources-virtual-machines-{{ vm }}-record-cmd-run:
+  cmd.run:
+    - env:
+      - ARVADOS_API_TOKEN: {{ api_token }}
+      - ARVADOS_API_HOST: {{ api_host }}
+    - name: |
+        arv --format=uuid \
+          virtual_machine \
+          create \
+          --virtual-machine '{"hostname":"{{ vm_name }}" }'
+    - onlyif: |
+        ARVADOS_API_TOKEN={{ api_token }} \
+        ARVADOS_API_HOST={{ api_host }} \
+        arv --short \
+          virtual_machine \
+          list \
+          --filters '[["hostname", "=", "{{ vm_name }}"]]' | \
+          /bin/grep -qE "fixme-2x53u-[a-z0-9_]{15}" && \
+          false
+{%- endfor %}

diff --git a/arvados/defaults.yaml b/arvados/defaults.yaml
index 07780b686eb745c03722fa4fe76ec947293f2ba3..bceb84eb71892fd49a3323991d025019ba869537 100644 (file)
--- a/arvados/defaults.yaml
+++ b/arvados/defaults.yaml
@@ -52,6 +52,8 @@ arvados:
       key: ''
       insecure: false
 
       key: ''
       insecure: false
 

+    resources: {}
+

   ### THESE ARE THE PACKAGES AND DAEMONS BASIC CONFIGS
   ##### API
   api:
   ### THESE ARE THE PACKAGES AND DAEMONS BASIC CONFIGS
   ##### API
   api:

diff --git a/kitchen.yml b/kitchen.yml
index cb3531930e8153f2ad7bfd575c497f2f6279a52e..a511ce8fe5df3b0625f9f53e85b70747d18889df 100644 (file)
--- a/kitchen.yml
+++ b/kitchen.yml
@@ -287,7 +287,7 @@ suites:
         - path: test/integration/keepproxy
         - path: test/integration/keepweb
         - path: test/integration/controller
         - path: test/integration/keepproxy
         - path: test/integration/keepweb
         - path: test/integration/controller

-  #### workbench,workbench2
+  #### workbench,workbench2,webshell

   - name: workbench
     driver:
       hostname: workbench.fixme.example.net
   - name: workbench
     driver:
       hostname: workbench.fixme.example.net


@@ -316,12 +316,14 @@ suites:
               - example_nginx
               - example_nginx_workbench
               - example_nginx_workbench2
               - example_nginx
               - example_nginx_workbench
               - example_nginx_workbench2

+              - example_nginx_webshell

       pillars_from_files:
         # yamllint disable rule:line-length
         arvados.sls: test/salt/pillar/arvados.sls
         example_nginx.sls: test/salt/pillar/examples/nginx_passenger.sls
         example_nginx_workbench.sls: test/salt/pillar/examples/nginx_workbench_configuration.sls
         example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls
       pillars_from_files:
         # yamllint disable rule:line-length
         arvados.sls: test/salt/pillar/arvados.sls
         example_nginx.sls: test/salt/pillar/examples/nginx_passenger.sls
         example_nginx_workbench.sls: test/salt/pillar/examples/nginx_workbench_configuration.sls
         example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls

+        example_nginx_webshell.sls: test/salt/pillar/examples/nginx_webshell_configuration.sls

         # yamllint enable rule:line-length
     verifier:
       inspec_tests:
         # yamllint enable rule:line-length
     verifier:
       inspec_tests:

diff --git a/pillar.example b/pillar.example
index e5278ddd07b63617e18f0b6511684394e8288ed2..ba5f228a1a7017f4975dbc81f4deb943518fb3d5 100644 (file)
--- a/pillar.example
+++ b/pillar.example
@@ -87,7 +87,6 @@ arvados:
       #   verify-ca: false
       #   client_encoding: UTF8
 
       #   verify-ca: false
       #   client_encoding: UTF8
 

-

     tls:
       # certificate: ''
       # key: ''
     tls:
       # certificate: ''
       # key: ''


@@ -114,6 +113,35 @@ arvados:
       keep_access_key: changemekeepaccesskey
       keep_secret_key: changemekeepsecretkey
 
       keep_access_key: changemekeepaccesskey
       keep_secret_key: changemekeepsecretkey
 

+    ### ARVADOS RESOURCES
+    # This dict allows you to create various resources in the Arvados
+    # database so they're ready to use.
+    # Check the `arvados.api.resources.* states to see which can be
+    # currently managed
+
+    ### SHELL / WEBSHELL REGISTRATION
+    # In order to use shell nodes via webshell, Arvados needs to know of
+    # their existence and they need to be configured as upstreams in nginx
+    # (see https://doc.arvados.org/v2.0/install/install-webshell.html)
+    # This could be achieved in various ways (ie, through salt mine if you
+    # want them to be dinamically created), but that's outside the scope of
+    # this formula. The following dict is just an example that will be used
+    # by the `arvados.api.resources.virtual_machines` state to add entries
+    # in Arvados' database of the cluster's resources'
+    # It's additionally used in the `test/salt/pillar/examples/nginx_webshell_configuration.sls`
+    # pillar to add the corresponding `location` entries in nginx's webshell vhosts & upstreams
+    resources:
+      virtual_machines:
+        shell1:
+          name: webshell1      # if not set, will match the one of the dict key above
+          backend: 1.2.3.4     # upstream host ip/name that has the shell role
+          port: 4200           # port where shellinabox is listening
+        # when no other parameter is set:
+        # `name` will match the name of the key
+        # backend` will match `name`
+        # `port` will default to shellinabox's 4200
+        webshell2: {}
+

     ### VOLUMES
     ## This should usually match all your `keepstore` instances
     Volumes:
     ### VOLUMES
     ## This should usually match all your `keepstore` instances
     Volumes:


@@ -225,13 +253,6 @@ arvados:
 #     service:
 #       name: keepstore
 #       port: 25107
 #     service:
 #       name: keepstore
 #       port: 25107

-#  #### GIT-HTTPD
-#   githttpd:
-#     pkg:
-#       name: arvados-git-httpd
-#     service:
-#       name: arvados-git-httpd
-#       port: 9001

 #  #### SHELL
 #   shell:
 #     pkg:
 #  #### SHELL
 #   shell:
 #     pkg:


@@ -269,13 +290,6 @@ arvados:
 #     service:
 #       name: arvados-ws
 #       port: 8005
 #     service:
 #       name: arvados-ws
 #       port: 8005

-#  #### SSO
-#   sso:
-#     pkg:
-#       name: arvados-sso
-#     service:
-#       name: arvados-sso
-#       port: 8900

 
 #  ## SALTSTACK FORMULAS TOFS configuration
 #   https://template-formula.readthedocs.io/en/latest/TOFS_pattern.html
 
 #  ## SALTSTACK FORMULAS TOFS configuration
 #   https://template-formula.readthedocs.io/en/latest/TOFS_pattern.html

diff --git a/test/integration/api/controls/resources_spec.rb b/test/integration/api/controls/resources_spec.rb
new file mode 100644 (file)
index 0000000..c4e2f6b
--- /dev/null
+++ b/test/integration/api/controls/resources_spec.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+test_cmd = <<~TEST_CMD
+  su -l kitchen -c \
+    "ARVADOS_API_TOKEN=\\"systemroottokenmushaveatleast32characters\\" \
+    ARVADOS_API_HOST=\\"fixme.example.net\\" \
+    arv virtual_machine list --filters '[[\\"hostname\\", \\"=\\", \\"%s\\"]]'"
+TEST_CMD
+
+control 'arvados api resources' do
+  impact 0.5
+  title 'should be created'
+
+  %w[
+    webshell1
+    webshell2
+  ].each do |vm|
+    describe "virtual machine #{vm}" do
+      subject do
+        command(test_cmd % vm)
+      end
+      its('stdout') { should match(/"uuid":"fixme-2x53u-[a-z0-9_]{15}"/) }
+      its('stdout') { should match(/"hostname":"#{vm}"/) }
+      its('stderr') { should eq '' }
+      its('exit_status') { should eq 0 }
+    end
+  end
+end

diff --git a/test/integration/shell/controls/config_spec.rb b/test/integration/shell/controls/config_spec.rb
index 54852fa5a19f930387f52390ae18dd594907bbdd..0ada81cdcb7dfbe0101c2a90ac76765410702831 100644 (file)
--- a/test/integration/shell/controls/config_spec.rb
+++ b/test/integration/shell/controls/config_spec.rb
@@ -4,7 +4,7 @@ control 'arvados configuration file' do
   title 'should not exist'
 
   describe file('/etc/arvados/config.yml') do
   title 'should not exist'
 
   describe file('/etc/arvados/config.yml') do

-    it { should_not exist}
+    it { should_not exist }

   end
 end
 
   end
 end
 

diff --git a/test/salt/pillar/arvados.sls b/test/salt/pillar/arvados.sls
index d8117c62537ae3676a4bcd87bf3bc6cda57d5148..1abe76c2499682c9b8501c25a748aac59bcb71fa 100644 (file)
--- a/test/salt/pillar/arvados.sls
+++ b/test/salt/pillar/arvados.sls
@@ -55,6 +55,14 @@ arvados:
       # required to test with snakeoil certs
       insecure: true
 
       # required to test with snakeoil certs
       insecure: true
 

+    resources:
+      virtual_machines:
+        shell1:
+          name: webshell1
+          backend: 1.2.3.4
+          port: 4200
+        webshell2: {}
+

     ### TOKENS
     tokens:
       system_root: systemroottokenmushaveatleast32characters
     ### TOKENS
     tokens:
       system_root: systemroottokenmushaveatleast32characters

diff --git a/test/salt/pillar/examples/nginx_webshell_configuration.sls b/test/salt/pillar/examples/nginx_webshell_configuration.sls
index 022cd36ac5635acb7ddad417ce7b90a3c1707cf9..e28fb9a67c3b036dfdd297d432a2db417cbfa729 100644 (file)
--- a/test/salt/pillar/examples/nginx_webshell_configuration.sls
+++ b/test/salt/pillar/examples/nginx_webshell_configuration.sls
@@ -7,6 +7,7 @@ nginx:
 
       ### STREAMS
       http:
 
       ### STREAMS
       http:

+        {%- for shell_node, params in %}

         upstream webshell_upstream:
           - server: 'shell.internal:4200 fail_timeout=10s'
 
         upstream webshell_upstream:
           - server: 'shell.internal:4200 fail_timeout=10s'