From fa49dbe833c7867ac95da84f9b36c8114cd89039 Mon Sep 17 00:00:00 2001 From: =?utf8?q?Javier=20B=C3=A9rtoli?= Date: Wed, 2 Dec 2020 10:30:23 -0300 Subject: [PATCH] docs(examples): better organization and naming Renamed some example files and directories, to better represent their purpose. Added references to the Arvados' provision script in the documentation. --- docs/README.rst | 22 +++++++++++++++-- kitchen.yml | 22 +++++++---------- test/salt/pillar/examples/README.rst | 7 ++++++ test/salt/pillar/examples/nginx_passenger.sls | 4 ++-- test/salt/pillar/examples/postgresql.sls | 4 ++-- .../states/examples/single_host/README.rst | 17 +++++++++++++ .../single_host/host_entries.sls} | 2 +- .../single_host/snakeoil_certs.sls} | 24 ++++++++++--------- 8 files changed, 70 insertions(+), 32 deletions(-) create mode 100644 test/salt/pillar/examples/README.rst create mode 100644 test/salt/states/examples/single_host/README.rst rename test/salt/states/{example_single_host_host_entries/init.sls => examples/single_host/host_entries.sls} (91%) rename test/salt/states/{example_add_snakeoil_certs/init.sls => examples/single_host/snakeoil_certs.sls} (61%) diff --git a/docs/README.rst b/docs/README.rst index e5982c7..d05d5e8 100644 --- a/docs/README.rst +++ b/docs/README.rst @@ -44,7 +44,22 @@ Please see `How to contribute `_ you can find `a provision script `_ +to deploy a single-node, all-in-one Arvados cluster (The script uses this formula to get a cluster up and running in Saltstack's master-less mode). + +The `single-node` install does not include SLURM: it is intended for an `all-in-one-host` installation, +so it uses `crunch-dispatch-local` to run containers in the same instance. + +The provision script can be run anywhere, so you can run it in an AWS instance and you'll get a `single-node` Arvados cluster there. + +The Arvados formula allows you to `install any dispatcher available `_, +provided you configure the pillars the way you need them. + +Arvados currently has three dispatchers: + +* **crunch-dispatch-local** (for single node installations), +* **arvados-dispatch-cloud** (for dynamic compute on AWS or Azure) and +* **crunch-dispatch-slurm** (for SLURM integration). Requisites ---------- @@ -56,7 +71,10 @@ We suggest you use the `postgres-formula `_ and the `letsencrypt-formula `_ to satisfy these dependencies. In the **test/salt/pillar/examples/** directory there are example pillar YAMLs to set up these packages, using the mentioned formulas -as Arvados needs them. +as Arvados needs them.a + +In the **test/salt/states/examples/** directory there are some example helper states to set up a few requirements for single-node +(all-in-one) Arvados host. Usage ----- diff --git a/kitchen.yml b/kitchen.yml index 4067e1c..f2e0add 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -104,16 +104,14 @@ suites: state_top: base: '*': - - example_single_host_host_entries - - example_add_snakeoil_certs + - single_host.host_entries + - single_host.snakeoil_certs - locale - nginx.passenger - postgres - arvados.repo - arvados.api - arvados.websocket - # keepproxy complains when using snakeoil certs, so we can't - # properly test it here until next version removes this limitation - arvados.keepproxy - arvados.keepweb - arvados.controller @@ -146,10 +144,8 @@ suites: example_nginx_controller.sls: test/salt/pillar/examples/nginx_controller_configuration.sls # yamllint enable rule:line-length dependencies: - - name: example_single_host_host_entries - path: test/salt/states - - name: example_add_snakeoil_certs - path: test/salt/states + - name: single_host + path: test/salt/states/examples - name: locale repo: git source: https://github.com/saltstack-formulas/locale-formula.git @@ -175,8 +171,8 @@ suites: state_top: base: '*': - - example_single_host_host_entries - - example_add_snakeoil_certs + - single_host.host_entries + - single_host.snakeoil_certs - nginx.passenger - arvados.repo - arvados.workbench @@ -197,10 +193,8 @@ suites: example_nginx_workbench2.sls: test/salt/pillar/examples/nginx_workbench2_configuration.sls # yamllint enable rule:line-length dependencies: - - name: example_single_host_host_entries - path: test/salt/states - - name: example_add_snakeoil_certs - path: test/salt/states + - name: single_host + path: test/salt/states/examples - name: nginx repo: git source: https://github.com/netmanagers/nginx-formula.git diff --git a/test/salt/pillar/examples/README.rst b/test/salt/pillar/examples/README.rst new file mode 100644 index 0000000..b0a7e85 --- /dev/null +++ b/test/salt/pillar/examples/README.rst @@ -0,0 +1,7 @@ +Pillar examples +=============== + +The files in this directory are pillar examples for the other formulas used to install +Arvados (`locale-formula `_, +`postgres-formula `_ and +`nginx-formula `_. diff --git a/test/salt/pillar/examples/nginx_passenger.sls b/test/salt/pillar/examples/nginx_passenger.sls index 8c41acb..ec79746 100644 --- a/test/salt/pillar/examples/nginx_passenger.sls +++ b/test/salt/pillar/examples/nginx_passenger.sls @@ -53,8 +53,8 @@ nginx: # - resolver: 127.0.0.1 ssl_snakeoil.conf: - - ssl_certificate: /etc/ssl/certs/ssl-cert-snakeoil.pem - - ssl_certificate_key: /etc/ssl/private/ssl-cert-snakeoil.key + - ssl_certificate: /etc/ssl/certs/arvados-snakeoil-cert.pem + - ssl_certificate_key: /etc/ssl/private/arvados-snakeoil-cert.key ### SITES servers: diff --git a/test/salt/pillar/examples/postgresql.sls b/test/salt/pillar/examples/postgresql.sls index aec4f13..5d800ec 100644 --- a/test/salt/pillar/examples/postgresql.sls +++ b/test/salt/pillar/examples/postgresql.sls @@ -7,8 +7,8 @@ postgres: postgresconf: |- listen_addresses = '*' # listen on all interfaces #ssl = on - #ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' - #ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' + #ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem' + #ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' acls: - ['local', 'all', 'postgres', 'peer'] - ['local', 'all', 'all', 'peer'] diff --git a/test/salt/states/examples/single_host/README.rst b/test/salt/states/examples/single_host/README.rst new file mode 100644 index 0000000..b50716d --- /dev/null +++ b/test/salt/states/examples/single_host/README.rst @@ -0,0 +1,17 @@ +Helper states for all-in-one setup +================================== + +These states are helpful for setting up an all-in-one Arvados host. + +* `host_entries.sls`: adds a bunch of host entries in the `/etc/hosts` file of + the host instance, so all Arvados' components can find each other correctly, + using meaningful names. + +* `snakeoil_certs.sls`: Arvados uses SSL/TLS for communications, so you'll need + certificates for the different hosts. If you can't provide valid certificates + issued by a recognized CA, this state will create a SnakeOil CA and issue + certificates signed by it. + + The certs can't be self-signed because some of the libraries that Arvados + uses require certs issued by a CA. For this reason, if you use this state, + you'll need to copy the created CA cert to your certificates' directory. diff --git a/test/salt/states/example_single_host_host_entries/init.sls b/test/salt/states/examples/single_host/host_entries.sls similarity index 91% rename from test/salt/states/example_single_host_host_entries/init.sls rename to test/salt/states/examples/single_host/host_entries.sls index 6425448..855757e 100644 --- a/test/salt/states/example_single_host_host_entries/init.sls +++ b/test/salt/states/examples/single_host/host_entries.sls @@ -3,7 +3,7 @@ {%- from "arvados/map.jinja" import arvados with context %} {%- set tpldir = curr_tpldir %} -arvados_hosts_entries: +arvados_test_salt_states_examples_single_host_etc_hosts_host_present: host.present: - ip: {{ grains.get('ipv4')[0] }} - names: diff --git a/test/salt/states/example_add_snakeoil_certs/init.sls b/test/salt/states/examples/single_host/snakeoil_certs.sls similarity index 61% rename from test/salt/states/example_add_snakeoil_certs/init.sls rename to test/salt/states/examples/single_host/snakeoil_certs.sls index 158abcc..b76bdce 100644 --- a/test/salt/states/example_add_snakeoil_certs/init.sls +++ b/test/salt/states/examples/single_host/snakeoil_certs.sls @@ -3,9 +3,11 @@ {%- from "arvados/map.jinja" import arvados with context %} {%- set tpldir = curr_tpldir %} -snake_oil_certs: +arvados_test_salt_states_examples_single_host_snakeoil_certs_openssl_pkg_installed: pkg.installed: - name: openssl + +arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run: cmd.run: - name: | cat > /tmp/openssl.cnf <<-CNF @@ -31,7 +33,6 @@ snake_oil_certs: {%- endfor %} {%- for entry in [ 'keep', - 'keep0', 'collections', 'download', 'ws', @@ -45,16 +46,16 @@ snake_oil_certs: mkdir -p /etc/ssl/certs/ /etc/ssl/private/ && \ openssl req -config /tmp/openssl.cnf -new -x509 -days 3650 -nodes -sha256 \ - -out /etc/ssl/certs/ssl-cert-snakeoil.pem \ - -keyout /etc/ssl/private/ssl-cert-snakeoil.key > /tmp/snake_oil_certs.output 2>&1 && \ - chmod 0644 /etc/ssl/certs/ssl-cert-snakeoil.pem && \ - chmod 0640 /etc/ssl/private/ssl-cert-snakeoil.key - - unless: test -f /etc/ssl/private/ssl-cert-snakeoil.key + -out /etc/ssl/certs/arvados-snakeoil-cert.pem \ + -keyout /etc/ssl/private/arvados-snakeoil-cert.key > /tmp/snake_oil_certs.output 2>&1 && \ + chmod 0644 /etc/ssl/certs/arvados-snakeoil-cert.pem && \ + chmod 0640 /etc/ssl/private/arvados-snakeoil-cert.key + - unless: test -f /etc/ssl/private/arvados-snakeoil-cert.key - require: - - pkg: openssl + - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_openssl_pkg_installed {%- if grains.get('os_family') == 'Debian' %} -ssl_certs: +arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed: pkg.installed: - name: ssl-cert - require_in: @@ -63,7 +64,8 @@ ssl_certs: snake_oil_certs_permissions: cmd.run: - name: | - chown root:ssl-cert /etc/ssl/private/ssl-cert-snakeoil.key + chown root:ssl-cert /etc/ssl/private/arvados-snakeoil-cert.key - require: - - pkg: ssl_certs + - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run + - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed {%- endif %} -- 2.30.2