tools/ansible/install-test-env.yml

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4 #
   5 # install-test-env.yml - Ansible playbook to set up a host to run Arvados tests
   6 #
   7 # After you run this playbook on a host, you should be able to clone the
   8 # Arvados git repository and run `build/run-tests.sh`.
   9 # As of April 2025 this has been tested to work on Debian 12.
  10 #
  11 # In order to use this playbook, you must write an Arvados `config.yml` with
  12 # PostgreSQL connection details for the `zzzzz` cluster.
  13 # `files/default-test-config.yml` has the default credentials used by tests.
  14 # You can copy that file and edit `dbname`, `user`, and `password` to taste.
  15 # Pass the path of your file to `ansible-playbook` with
  16 # `-e arvados_config_file=config.yml`.
  17
  18 - name: Load Arvados configuration
  19   hosts: all
  20   tasks:
  21     - name: Load Arvados configuration file
  22       delegate_to: localhost
  23       ansible.builtin.include_vars:
  24         name: arvados_config
  25         file: "{{ arvados_config_file }}"
  26     - name: Load Arvados cluster configuration
  27       ansible.builtin.set_fact:
  28         arvados_cluster: "{{ arvados_config.Clusters.zzzzz }}"
  29       failed_when: arvados_cluster is undefined
  30
  31 - hosts: all
  32   vars:
  33     arvados_dev_user: "{{ ansible_user_id }}"
  34     arvados_nodejs_version: "14.21.3"
  35     arvados_nodejs_destdir: "/opt/node-v{{ arvados_nodejs_version }}-linux-x64"
  36
  37   tasks:
  38     - name: Bootstrap packages required for Ansible
  39       become: yes
  40       ansible.builtin.raw: "apt-get -o DPkg::Lock::Timeout=300 -qy {{ item }}"
  41       loop:
  42         - update
  43         - install acl gnupg python3-apt python3-debian python3-psycopg2 xz-utils
  44     - name: Install all required development packages
  45       become: yes
  46       ansible.builtin.apt:
  47         name:
  48           # Common build and test tools
  49           - bison
  50           - ca-certificates
  51           - curl
  52           - diffutils
  53           - g++
  54           - git
  55           - jq
  56           - libnss-systemd
  57           - make
  58           - nginx
  59           - postgresql-client
  60           - python3-dev
  61           - python3-venv
  62           - ruby
  63           - ruby-dev
  64           # Common libraries
  65           - libcurl4-openssl-dev
  66           - libssl-dev
  67           # build/run-tests.sh
  68           - bsdextrautils
  69           - net-tools
  70           # build/run-build-packages*.sh
  71           - createrepo-c
  72           - dpkg-dev
  73           # doc
  74           - linkchecker
  75           # lib/controller
  76           - rsync
  77           # lib/pam
  78           - libpam-dev
  79           # sdk/java-v2
  80           - default-jdk-headless
  81           - gradle
  82           # sdk/R
  83           - libfontconfig1-dev
  84           - libfreetype6-dev
  85           - libfribidi-dev
  86           - libharfbuzz-dev
  87           - libjpeg-dev
  88           - libpng-dev
  89           - libtiff5-dev
  90           - libxml2-dev
  91           - r-base
  92           - r-cran-testthat
  93           # services/api
  94           - libnsl2
  95           - libpq-dev
  96           - libyaml-dev
  97           - locales
  98           - procps
  99           - shared-mime-info
 100           - zlib1g-dev
 101           # services/fuse
 102           - fuse
 103           - libfuse-dev
 104           # services/keep-web
 105           - cadaver
 106           - mime-support
 107           # services/workbench2
 108           # <https://docs.cypress.io/app/get-started/install-cypress#Linux-Prerequisites>
 109           - firefox-esr
 110           - libasound2
 111           - libgbm-dev
 112           - libgtk-3-0
 113           - libgtk2.0-0
 114           - libnotify-dev
 115           - libnss3
 116           - libxss1
 117           - libxtst6
 118           - xauth
 119           - xvfb
 120
 121     - ansible.builtin.include_role:
 122         name: arvados_go
 123     - ansible.builtin.include_role:
 124         name: arvados_compute
 125       vars:
 126         arvados_compute_packages: []
 127         arvados_compute_docker: true
 128         arvados_compute_singularity: true
 129
 130     # RailsAPI tests depend on the en_US.UTF-8 locale.
 131     # It must be generated before starting the PostgreSQL server.
 132     - name: Configure en_US.UTF-8 locale
 133       become: yes
 134       ansible.builtin.lineinfile:
 135         path: /etc/locale.gen
 136         regexp: "^[# ]*en_US.UTF-8 +UTF-8 *$"
 137         line: en_US.UTF-8 UTF-8
 138       register: locale_gen
 139     - name: Run locale-gen
 140       when: locale_gen.changed
 141       become: yes
 142       ansible.builtin.command:
 143         cmd: locale-gen
 144     - ansible.builtin.include_role:
 145         name: arvados_postgresql
 146       vars:
 147         arvados_postgresql_hba_sources:
 148           - "127.0.0.1/24"
 149           - "::1/128"
 150
 151     # Workbench build dependencies
 152     - name: Install Node.js
 153       become: yes
 154       ansible.builtin.unarchive:
 155         src: "https://nodejs.org/dist/v{{ arvados_nodejs_version }}/node-v{{ arvados_nodejs_version }}-linux-x64.tar.xz"
 156         dest: "{{ arvados_nodejs_destdir|dirname }}"
 157         remote_src: yes
 158         creates: "{{ (arvados_nodejs_destdir, 'bin/node')|path_join }}"
 159     - name: Install yarn
 160       become: yes
 161       ansible.builtin.command:
 162         cmd: npm install -g yarn
 163         creates: "{{ (arvados_nodejs_destdir, 'bin/yarn')|path_join }}"
 164       environment:
 165         PATH: "{{ (arvados_nodejs_destdir, 'bin')|path_join }}:{{ ansible_env.PATH }}"
 166     - name: Add Node commands to PATH
 167       become: yes
 168       ansible.builtin.file:
 169         state: link
 170         src: "{{ (arvados_nodejs_destdir, 'bin', item)|path_join }}"
 171         dest: "{{ ('/usr/local/bin', item)|path_join }}"
 172       loop:
 173         - node
 174         - npm
 175         - yarn
 176         - yarnpkg
 177     # Configuration to run Workbench
 178     - name: Check fs.inotify.max_user_watches sysctl value
 179       ansible.builtin.command:
 180         cmd: /sbin/sysctl --values fs.inotify.max_user_watches
 181       register: max_user_watches_value
 182     - name: Increase fs.inotify.max_user_watches
 183       vars:
 184         max_user_watches_wanted: 524288
 185       when: "max_user_watches_value.stdout|int < max_user_watches_wanted"
 186       become: yes
 187       ansible.builtin.command:
 188         argv:
 189           - sysctl
 190           - "fs.inotify.max_user_watches={{ max_user_watches_wanted }}"
 191       register: max_user_watches_set
 192     - name: Set fs.inotify.max_user_watches permanently
 193       when: max_user_watches_set.changed
 194       become: yes
 195       ansible.builtin.copy:
 196         content: |
 197           ### This file is managed by Ansible
 198           # React sets many inotify watchers and needs the limit increased.
 199           {{ max_user_watches_set.stdout }}
 200         dest: /etc/sysctl.d/arvados-workbench.conf
 201         owner: root
 202         group: root
 203         mode: 0644
 204
 205     - ansible.builtin.include_role:
 206         name: arvados_database
 207       vars:
 208         arvados_database_login_host: ""
 209         # Let the test user drop and recreate the database wholesale
 210         arvados_database_role_attr_flags: CREATEDB
 211
 212     - name: Set up Arvados development user
 213       become: yes
 214       ansible.builtin.user:
 215         name: "{{ arvados_dev_user }}"
 216         groups:
 217           - docker
 218         append: yes
 219       register: dev_user
 220     - name: Set up .config/arvados
 221       become: yes
 222       become_user: "{{ arvados_dev_user }}"
 223       ansible.builtin.file:
 224         path: "{{ (dev_user.home, item)|path_join }}"
 225         state: directory
 226       loop:
 227         - .config
 228         - .config/arvados
 229     - name: Write arvados/config.yml for testing
 230       become: yes
 231       become_user: "{{ arvados_dev_user }}"
 232       ansible.builtin.copy:
 233         src: "{{ arvados_config_file }}"
 234         dest: "{{ (dev_user.home, '.config/arvados/config.yml')|path_join }}"
 235         mode: 0600
 236     - name: Add Arvados test configuration to profile.d
 237       become: yes
 238       ansible.builtin.copy:
 239         content: |
 240           if [ -z "${CONFIGSRC:-}" ] && [ -e ~/.config/arvados/config.yml ]; then
 241             export CONFIGSRC="$HOME/.config/arvados"
 242           fi
 243         dest: /etc/profile.d/arvados-test.sh