tools/ansible/install-test-env.yml

   1 # Copyright (C) The Arvados Authors. All rights reserved.
   2 #
   3 # SPDX-License-Identifier: Apache-2.0
   4 #
   5 # install-test-env.yml - Ansible playbook to set up a host to run Arvados tests
   6 #
   7 # After you run this playbook on a host, you should be able to clone the
   8 # Arvados git repository and run `build/run-tests.sh`.
   9 # As of April 2025 this has been tested to work on Debian 12.
  10 #
  11 # In order to use this playbook, you must write an Arvados `config.yml` with
  12 # PostgreSQL connection details for the `zzzzz` cluster.
  13 # `files/default-test-config.yml` has the default credentials used by tests.
  14 # You can copy that file and edit `dbname`, `user`, and `password` to taste.
  15 # Pass the path of your file to `ansible-playbook` with
  16 # `-e arvados_config_file=config.yml`.
  17
  18 - name: Load Arvados configuration
  19   hosts: all
  20   tasks:
  21     - name: Load Arvados configuration file
  22       delegate_to: localhost
  23       ansible.builtin.include_vars:
  24         name: arvados_config
  25         file: "{{ arvados_config_file }}"
  26     - name: Load Arvados cluster configuration
  27       ansible.builtin.set_fact:
  28         arvados_cluster: "{{ arvados_config.Clusters.zzzzz }}"
  29       failed_when: arvados_cluster is undefined
  30
  31 - hosts: all
  32   vars:
  33     arvados_dev_user: "{{ ansible_user_id }}"
  34     arvados_nodejs_version: "14.21.3"
  35     arvados_nodejs_destdir: "/opt/node-v{{ arvados_nodejs_version }}-linux-x64"
  36
  37   tasks:
  38     - name: Bootstrap packages required for Ansible
  39       become: yes
  40       ansible.builtin.raw: "apt-get -o DPkg::Lock::Timeout=300 -qy {{ item }}"
  41       loop:
  42         - update
  43         - install acl gnupg python3-apt python3-debian xz-utils
  44     - name: Install all required development packages
  45       become: yes
  46       ansible.builtin.apt:
  47         name:
  48           # Common build and test tools
  49           - bison
  50           - ca-certificates
  51           - curl
  52           - diffutils
  53           - g++
  54           - git
  55           - jq
  56           - libnss-systemd
  57           - make
  58           - nginx
  59           - postgresql-client
  60           - python3-dev
  61           - python3-venv
  62           - ruby
  63           - ruby-dev
  64           # Common libraries
  65           - libcurl4-openssl-dev
  66           - libssl-dev
  67           # build/run-tests.sh
  68           - bsdextrautils
  69           - net-tools
  70           # build/run-build-packages*.sh
  71           - createrepo-c
  72           - dpkg-dev
  73           # doc
  74           - linkchecker
  75           # lib/controller
  76           - rsync
  77           # lib/pam
  78           - libpam-dev
  79           # sdk/java-v2
  80           - default-jdk-headless
  81           - gradle
  82           # sdk/R
  83           - libfontconfig1-dev
  84           - libfreetype6-dev
  85           - libfribidi-dev
  86           - libharfbuzz-dev
  87           - libjpeg-dev
  88           - libpng-dev
  89           - libtiff5-dev
  90           - libxml2-dev
  91           - r-base
  92           - r-cran-testthat
  93           # services/api
  94           - libnsl2
  95           - libpq-dev
  96           - libyaml-dev
  97           - locales
  98           - procps
  99           - shared-mime-info
 100           - zlib1g-dev
 101           # services/fuse
 102           - fuse
 103           - libfuse-dev
 104           # services/keep-web
 105           - cadaver
 106           - mime-support
 107           # services/workbench2
 108           # <https://docs.cypress.io/app/get-started/install-cypress#Linux-Prerequisites>
 109           - firefox-esr
 110           - libasound2
 111           - libgbm-dev
 112           - libgtk-3-0
 113           - libgtk2.0-0
 114           - libnotify-dev
 115           - libnss3
 116           - libxss1
 117           - libxtst6
 118           - xauth
 119           - xvfb
 120
 121     - ansible.builtin.include_role:
 122         name: arvados_go
 123     - ansible.builtin.include_role:
 124         name: arvados_compute
 125       vars:
 126         arvados_compute_packages: []
 127         arvados_compute_docker: true
 128         arvados_compute_singularity: true
 129
 130     # RailsAPI tests depend on the en_US.UTF-8 locale.
 131     # It must be generated before starting the PostgreSQL server.
 132     - name: Configure en_US.UTF-8 locale
 133       become: yes
 134       ansible.builtin.lineinfile:
 135         path: /etc/locale.gen
 136         regexp: "^[# ]*en_US.UTF-8 +UTF-8 *$"
 137         line: en_US.UTF-8 UTF-8
 138       register: locale_gen
 139     - name: Run locale-gen
 140       when: locale_gen.changed
 141       become: yes
 142       ansible.builtin.command:
 143         cmd: locale-gen
 144     - ansible.builtin.include_role:
 145         name: arvados_postgresql
 146       vars:
 147         arvados_postgresql_config: {}
 148         arvados_postgresql_hba_sources:
 149           - "127.0.0.1/24"
 150           - "::1/128"
 151
 152     # Workbench build dependencies
 153     - name: Install Node.js
 154       become: yes
 155       ansible.builtin.unarchive:
 156         src: "https://nodejs.org/dist/v{{ arvados_nodejs_version }}/node-v{{ arvados_nodejs_version }}-linux-x64.tar.xz"
 157         dest: "{{ arvados_nodejs_destdir|dirname }}"
 158         remote_src: yes
 159         creates: "{{ (arvados_nodejs_destdir, 'bin/node')|path_join }}"
 160     - name: Install yarn
 161       become: yes
 162       ansible.builtin.command:
 163         cmd: npm install -g yarn
 164         creates: "{{ (arvados_nodejs_destdir, 'bin/yarn')|path_join }}"
 165       environment:
 166         PATH: "{{ (arvados_nodejs_destdir, 'bin')|path_join }}:{{ ansible_env.PATH }}"
 167     - name: Add Node commands to PATH
 168       become: yes
 169       ansible.builtin.file:
 170         state: link
 171         src: "{{ (arvados_nodejs_destdir, 'bin', item)|path_join }}"
 172         dest: "{{ ('/usr/local/bin', item)|path_join }}"
 173       loop:
 174         - node
 175         - npm
 176         - yarn
 177         - yarnpkg
 178     # Configuration to run Workbench
 179     - name: Check fs.inotify.max_user_watches sysctl value
 180       ansible.builtin.command:
 181         cmd: /sbin/sysctl --values fs.inotify.max_user_watches
 182       register: max_user_watches_value
 183     - name: Increase fs.inotify.max_user_watches
 184       vars:
 185         max_user_watches_wanted: 524288
 186       when: "max_user_watches_value.stdout|int < max_user_watches_wanted"
 187       become: yes
 188       ansible.builtin.command:
 189         argv:
 190           - sysctl
 191           - "fs.inotify.max_user_watches={{ max_user_watches_wanted }}"
 192       register: max_user_watches_set
 193     - name: Set fs.inotify.max_user_watches permanently
 194       when: max_user_watches_set.changed
 195       become: yes
 196       ansible.builtin.copy:
 197         content: |
 198           ### This file is managed by Ansible
 199           # React sets many inotify watchers and needs the limit increased.
 200           {{ max_user_watches_set.stdout }}
 201         dest: /etc/sysctl.d/arvados-workbench.conf
 202         owner: root
 203         group: root
 204         mode: 0644
 205
 206     - ansible.builtin.include_role:
 207         name: arvados_database
 208       vars:
 209         arvados_database_login_host: ""
 210         # Let the test user drop and recreate the database wholesale
 211         arvados_database_role_attr_flags: CREATEDB
 212
 213     - name: Set up Arvados development user
 214       become: yes
 215       ansible.builtin.user:
 216         name: "{{ arvados_dev_user }}"
 217         groups:
 218           - docker
 219         append: yes
 220       register: dev_user
 221     - name: Set up .config/arvados
 222       become: yes
 223       become_user: "{{ arvados_dev_user }}"
 224       ansible.builtin.file:
 225         path: "{{ (dev_user.home, item)|path_join }}"
 226         state: directory
 227       loop:
 228         - .config
 229         - .config/arvados
 230     - name: Write arvados/config.yml for testing
 231       become: yes
 232       become_user: "{{ arvados_dev_user }}"
 233       ansible.builtin.copy:
 234         src: "{{ arvados_config_file }}"
 235         dest: "{{ (dev_user.home, '.config/arvados/config.yml')|path_join }}"
 236         mode: 0600
 237     - name: Add Arvados test configuration to profile.d
 238       become: yes
 239       ansible.builtin.copy:
 240         content: |
 241           if [ -z "${CONFIGSRC:-}" ] && [ -e ~/.config/arvados/config.yml ]; then
 242             export CONFIGSRC="$HOME/.config/arvados"
 243           fi
 244         dest: /etc/profile.d/arvados-test.sh