From: Tom Clegg Date: Mon, 21 Mar 2022 14:17:29 +0000 (-0400) Subject: Merge branch '18691-freeze-project' X-Git-Tag: 2.4.0~36 X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/44c93373e97da98645d41ae8f09c6eef6788bb26?hp=7b070fc8458f4108d44d6bfb939e36d3cc76af84 Merge branch '18691-freeze-project' refs #18691 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/.licenseignore b/.licenseignore index e3289aa7c7..97ce38af93 100644 --- a/.licenseignore +++ b/.licenseignore @@ -87,4 +87,5 @@ sdk/python/tests/fed-migrate/*.cwl sdk/python/tests/fed-migrate/*.cwlex doc/install/*.xlsx sdk/cwl/tests/wf/hello.txt -sdk/cwl/tests/wf/indir1/hello2.txt \ No newline at end of file +sdk/cwl/tests/wf/indir1/hello2.txt +sdk/cwl/tests/chipseq/data/Genomes/* \ No newline at end of file diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock index b82568e7d2..ab4f3a173a 100644 --- a/apps/workbench/Gemfile.lock +++ b/apps/workbench/Gemfile.lock @@ -16,43 +16,43 @@ GEM remote: https://rubygems.org/ specs: RedCloth (4.3.2) - actioncable (5.2.6) - actionpack (= 5.2.6) + actioncable (5.2.6.3) + actionpack (= 5.2.6.3) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.6) - actionpack (= 5.2.6) - actionview (= 5.2.6) - activejob (= 5.2.6) + actionmailer (5.2.6.3) + actionpack (= 5.2.6.3) + actionview (= 5.2.6.3) + activejob (= 5.2.6.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.6) - actionview (= 5.2.6) - activesupport (= 5.2.6) + actionpack (5.2.6.3) + actionview (= 5.2.6.3) + activesupport (= 5.2.6.3) rack (~> 2.0, >= 2.0.8) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.6) - activesupport (= 5.2.6) + actionview (5.2.6.3) + activesupport (= 5.2.6.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.3) - activejob (5.2.6) - activesupport (= 5.2.6) + activejob (5.2.6.3) + activesupport (= 5.2.6.3) globalid (>= 0.3.6) - activemodel (5.2.6) - activesupport (= 5.2.6) - activerecord (5.2.6) - activemodel (= 5.2.6) - activesupport (= 5.2.6) + activemodel (5.2.6.3) + activesupport (= 5.2.6.3) + activerecord (5.2.6.3) + activemodel (= 5.2.6.3) + activesupport (= 5.2.6.3) arel (>= 9.0) - activestorage (5.2.6) - actionpack (= 5.2.6) - activerecord (= 5.2.6) + activestorage (5.2.6.3) + actionpack (= 5.2.6.3) + activerecord (= 5.2.6.3) marcel (~> 1.0.0) - activesupport (5.2.6) + activesupport (5.2.6.3) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -122,8 +122,8 @@ GEM multipart-post (>= 1.2, < 3) ffi (1.10.0) flamegraph (0.9.5) - globalid (0.4.2) - activesupport (>= 4.2.0) + globalid (1.0.0) + activesupport (>= 5.0) googleauth (0.9.0) faraday (~> 0.12) jwt (>= 1.4, < 3.0) @@ -150,20 +150,20 @@ GEM railties (>= 4) request_store (~> 1.0) logstash-event (1.2.02) - loofah (2.10.0) + loofah (2.14.0) crass (~> 1.0.2) nokogiri (>= 1.5.9) mail (2.7.1) mini_mime (>= 0.1.1) - marcel (1.0.1) + marcel (1.0.2) memoist (0.16.2) metaclass (0.0.4) method_source (1.0.0) mime-types (3.2.2) mime-types-data (~> 3.2015) mime-types-data (3.2019.0331) - mini_mime (1.1.0) - mini_portile2 (2.6.1) + mini_mime (1.1.2) + mini_portile2 (2.8.0) minitest (5.10.3) mocha (1.8.0) metaclass (~> 0.0.1) @@ -178,9 +178,9 @@ GEM net-ssh (5.2.0) net-ssh-gateway (2.0.0) net-ssh (>= 4.0.0) - nio4r (2.5.7) - nokogiri (1.12.5) - mini_portile2 (~> 2.6.1) + nio4r (2.5.8) + nokogiri (1.13.3) + mini_portile2 (~> 2.8.0) racc (~> 1.4) npm-rails (0.2.1) rails (>= 3.2) @@ -205,18 +205,18 @@ GEM rack (>= 1.2.0) rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.6) - actioncable (= 5.2.6) - actionmailer (= 5.2.6) - actionpack (= 5.2.6) - actionview (= 5.2.6) - activejob (= 5.2.6) - activemodel (= 5.2.6) - activerecord (= 5.2.6) - activestorage (= 5.2.6) - activesupport (= 5.2.6) + rails (5.2.6.3) + actioncable (= 5.2.6.3) + actionmailer (= 5.2.6.3) + actionpack (= 5.2.6.3) + actionview (= 5.2.6.3) + activejob (= 5.2.6.3) + activemodel (= 5.2.6.3) + activerecord (= 5.2.6.3) + activestorage (= 5.2.6.3) + activesupport (= 5.2.6.3) bundler (>= 1.3.0) - railties (= 5.2.6) + railties (= 5.2.6.3) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.4) actionpack (>= 5.0.1.x) @@ -225,16 +225,16 @@ GEM rails-dom-testing (2.0.3) activesupport (>= 4.2.0) nokogiri (>= 1.6) - rails-html-sanitizer (1.3.0) + rails-html-sanitizer (1.4.2) loofah (~> 2.3) rails-perftest (0.0.7) - railties (5.2.6) - actionpack (= 5.2.6) - activesupport (= 5.2.6) + railties (5.2.6.3) + actionpack (= 5.2.6.3) + activesupport (= 5.2.6.3) method_source rake (>= 0.8.7) thor (>= 0.19.0, < 2.0) - rake (13.0.3) + rake (13.0.6) raphael-rails (2.1.2) rb-fsevent (0.10.3) rb-inotify (0.10.0) @@ -288,7 +288,7 @@ GEM activesupport (>= 4.0) sprockets (>= 3.0.0) sshkey (2.0.0) - thor (1.1.0) + thor (1.2.1) thread_safe (0.3.6) tilt (2.0.9) tzinfo (1.2.9) @@ -296,7 +296,7 @@ GEM uglifier (2.7.2) execjs (>= 0.3.0) json (>= 1.8.0) - websocket-driver (0.7.4) + websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.5) xpath (2.1.0) diff --git a/apps/workbench/app/views/users/link_account.html.erb b/apps/workbench/app/views/users/link_account.html.erb index 86a0446e76..e45073e288 100644 --- a/apps/workbench/app/views/users/link_account.html.erb +++ b/apps/workbench/app/views/users/link_account.html.erb @@ -75,6 +75,8 @@ SPDX-License-Identifier: AGPL-3.0 %> }); <% end %> +<% if Rails.configuration.Login.LoginCluster.empty? %> + + +<% else %> +
+Self-serve account linking is not supported on this cluster. Please contact your Arvados administrator.
+<% end %> diff --git a/build/package-build-dockerfiles/build-all-build-containers.sh b/build/package-build-dockerfiles/build-all-build-containers.sh index 5ed33dc9f3..5f8817f20a 100755 --- a/build/package-build-dockerfiles/build-all-build-containers.sh +++ b/build/package-build-dockerfiles/build-all-build-containers.sh @@ -12,7 +12,7 @@ for target in `find -maxdepth 1 -type d |grep -v generated`; do target=${target#./} echo $target cd $target - docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE . + docker build --tag arvados/build:$target --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) . cd .. done diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile index 14a28901cb..e44d231edf 100644 --- a/build/package-build-dockerfiles/centos7/Dockerfile +++ b/build/package-build-dockerfiles/centos7/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH FROM centos:7 as build_x86_64 # Install go @@ -36,8 +37,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 @@ -64,7 +65,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile index efff0acc93..ed0a0cdc1f 100644 --- a/build/package-build-dockerfiles/debian10/Dockerfile +++ b/build/package-build-dockerfiles/debian10/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH ## dont use debian:10 here since the word 'buster' is used for rvm precompiled binaries FROM debian:buster as build_x86_64 @@ -41,8 +42,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 @@ -51,7 +52,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile index 54a6a0ec1d..cfeaf2463a 100644 --- a/build/package-build-dockerfiles/debian11/Dockerfile +++ b/build/package-build-dockerfiles/debian11/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH ## dont use debian:11 here since the word 'bullseye' is used for rvm precompiled binaries FROM debian:bullseye as build_x86_64 @@ -46,8 +47,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 @@ -56,7 +57,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile index ed2ca49541..9b20b41a4e 100644 --- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH FROM ubuntu:bionic as build_x86_64 # Install go @@ -40,8 +41,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 @@ -50,7 +51,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile index 58b4bc1ed8..f28e6fef1d 100644 --- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile @@ -3,6 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 ARG HOSTTYPE +ARG BRANCH FROM ubuntu:focal as build_x86_64 # Install go @@ -51,8 +52,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 -j $(grep -c processor /proc/cpuinfo) && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 @@ -61,7 +62,12 @@ RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(gr # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ ENV MAKE "make --jobs $(grep -c processor /proc/cpuinfo)" +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ cd /tmp/arvados/services/api && \ /usr/local/rvm/bin/rvm-exec default bundle install && \ cd /tmp/arvados/apps/workbench && \ diff --git a/build/package-test-dockerfiles/centos7/Dockerfile b/build/package-test-dockerfiles/centos7/Dockerfile index f83941824e..1010ef8c43 100644 --- a/build/package-test-dockerfiles/centos7/Dockerfile +++ b/build/package-test-dockerfiles/centos7/Dockerfile @@ -15,8 +15,8 @@ RUN touch /var/lib/rpm/* && \ gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.9 # Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612 diff --git a/build/package-test-dockerfiles/debian10/Dockerfile b/build/package-test-dockerfiles/debian10/Dockerfile index 3f9393ee55..e4b79930e8 100644 --- a/build/package-test-dockerfiles/debian10/Dockerfile +++ b/build/package-test-dockerfiles/debian10/Dockerfile @@ -17,8 +17,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 # udev daemon can't start in a container, so don't try. diff --git a/build/package-test-dockerfiles/debian11/Dockerfile b/build/package-test-dockerfiles/debian11/Dockerfile index 7cc543cf0d..8c91ca5c74 100644 --- a/build/package-test-dockerfiles/debian11/Dockerfile +++ b/build/package-test-dockerfiles/debian11/Dockerfile @@ -17,8 +17,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> /etc/gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 diff --git a/build/package-test-dockerfiles/ubuntu1804/Dockerfile b/build/package-test-dockerfiles/ubuntu1804/Dockerfile index 7347f32c8f..64894d799d 100644 --- a/build/package-test-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-test-dockerfiles/ubuntu1804/Dockerfile @@ -17,8 +17,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 # udev daemon can't start in a container, so don't try. diff --git a/build/package-test-dockerfiles/ubuntu2004/Dockerfile b/build/package-test-dockerfiles/ubuntu2004/Dockerfile index 061c8848ee..df1e71e75a 100644 --- a/build/package-test-dockerfiles/ubuntu2004/Dockerfile +++ b/build/package-test-dockerfiles/ubuntu2004/Dockerfile @@ -17,8 +17,8 @@ ADD generated/pkuczynski.asc /tmp/ RUN gpg --import --no-tty /tmp/mpapis.asc && \ gpg --import --no-tty /tmp/pkuczynski.asc && \ curl -L https://get.rvm.io | bash -s stable && \ - /usr/local/rvm/bin/rvm install 2.5 && \ - /usr/local/rvm/bin/rvm alias create default ruby-2.5 && \ + /usr/local/rvm/bin/rvm install 2.7 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 # udev daemon can't start in a container, so don't try. diff --git a/build/package-testing/test-package-python3-arvados-python-client.sh b/build/package-testing/test-package-python3-arvados-python-client.sh index 69f728c10e..1e294fe0a8 100755 --- a/build/package-testing/test-package-python3-arvados-python-client.sh +++ b/build/package-testing/test-package-python3-arvados-python-client.sh @@ -7,7 +7,9 @@ set -e arv-put --version >/dev/null -/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF +PYTHON=`ls /usr/share/python3*/dist/python3-arvados-python-client/bin/python3 |head -n1` + +$PYTHON << EOF import arvados print("Successfully imported arvados") EOF diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh index e06a732979..c1cc2e5877 100755 --- a/build/run-build-packages-one-target.sh +++ b/build/run-build-packages-one-target.sh @@ -195,7 +195,7 @@ fi echo $TARGET cd $TARGET -time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE . +time docker build --tag "$IMAGE" --build-arg HOSTTYPE=$HOSTTYPE --build-arg BRANCH=$(git rev-parse --abbrev-ref HEAD) . popd if test -z "$packages" ; then @@ -307,16 +307,24 @@ else set +e mv -f ${WORKSPACE}/packages/${TARGET}/* ${WORKSPACE}/packages/${TARGET}/processed/ 2>/dev/null set -e + # give bundle (almost) all the cores. See also the MAKE env var that is passed into the + # docker run command below. + # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ + tmpfile=$(mktemp /tmp/run-build-packages-one-target.XXXXXX) + cores=$(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) + printf -- "---\nBUNDLE_JOBS: \"$cores\"" > $tmpfile # Build packages. if docker run \ --rm \ "${docker_volume_args[@]}" \ + -v $tmpfile:/root/.bundle/config \ --env ARVADOS_BUILDING_VERSION="$ARVADOS_BUILDING_VERSION" \ --env ARVADOS_BUILDING_ITERATION="$ARVADOS_BUILDING_ITERATION" \ --env ARVADOS_DEBUG=$ARVADOS_DEBUG \ --env "ONLY_BUILD=$ONLY_BUILD" \ --env "FORCE_BUILD=$FORCE_BUILD" \ --env "ARCH=$ARCH" \ + --env "MAKE=make --jobs $cores" \ "$IMAGE" $COMMAND then echo @@ -325,6 +333,8 @@ else FINAL_EXITCODE=$? echo "ERROR: build packages on $IMAGE failed with exit status $FINAL_EXITCODE" >&2 fi + # Clean up the bundle config file + rm -f $tmpfile fi if test -n "$package_fails" ; then diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh index e36c4e88c0..aa4acb6a2b 100755 --- a/build/run-build-test-packages-one-target.sh +++ b/build/run-build-test-packages-one-target.sh @@ -15,6 +15,11 @@ Syntax: Build only a specific package (or ONLY_BUILD from environment) --arch Build a specific architecture (or ARCH from environment, defaults to native architecture) +--force-build + Build even if the package exists upstream or if it has already been + built locally +--force-test + Test even if there is no new untested package --upload If the build and test steps are successful, upload the packages to a remote apt repository (default: false) @@ -48,7 +53,7 @@ if ! [[ -d "$WORKSPACE" ]]; then fi PARSEDOPTS=$(getopt --name "$0" --longoptions \ - help,debug,upload,rc,target:,only-build:,arch:,build-version: \ + help,debug,upload,rc,target:,force-test,only-build:,force-build,arch:,build-version: \ -- "" "$@") if [ $? -ne 0 ]; then exit 1 @@ -72,6 +77,12 @@ while [ $# -gt 0 ]; do --target) TARGET="$2"; shift ;; + --force-test) + FORCE_TEST=1 + ;; + --force-build) + FORCE_BUILD=1 + ;; --only-build) ONLY_BUILD="$2"; shift ;; @@ -107,6 +118,14 @@ if [[ -n "$ONLY_BUILD" ]]; then build_args+=(--only-build "$ONLY_BUILD") fi +if [[ -n "$FORCE_BUILD" ]]; then + build_args+=(--force-build) +fi + +if [[ -n "$FORCE_TEST" ]]; then + build_args+=(--force-test) +fi + if [[ -n "$ARCH" ]]; then build_args+=(--arch "$ARCH") fi diff --git a/build/run-library.sh b/build/run-library.sh index 2a869553d1..fa2be6ac7a 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -492,7 +492,8 @@ handle_rails_package() { cd "$srcdir" mkdir -p tmp git rev-parse HEAD >git-commit.version - bundle package --all + bundle config set cache_all true + bundle package ) if [[ 0 != "$?" ]] || ! cd "$WORKSPACE/packages/$TARGET"; then echo "ERROR: $pkgname package prep failed" >&2 @@ -578,12 +579,12 @@ handle_workbench () { # Build the workbench server package test_rails_package_presence arvados-workbench "$WORKSPACE/apps/workbench" if [[ "$?" == "0" ]] ; then + calculate_go_package_version arvados_server_version cmd/arvados-server + arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") + ( set -e - calculate_go_package_version arvados_server_version cmd/arvados-server - arvados_server_iteration=$(default_iteration "arvados-server" "$arvados_server_version" "go") - # The workbench package has a build-time dependency on the arvados-server # package for config manipulation, so install it first. cd $WORKSPACE/cmd/arvados-server @@ -603,7 +604,8 @@ handle_workbench () { # We need to bundle to be ready even when we build a package without vendor directory # because asset compilation requires it. - bundle install --system >"$STDOUT_IF_DEBUG" + bundle config set --local system 'true' >"$STDOUT_IF_DEBUG" + bundle install >"$STDOUT_IF_DEBUG" # clear the tmp directory; the asset generation step will recreate tmp/cache/assets, # and we want that in the package, so it's easier to not exclude the tmp directory diff --git a/doc/_config.yml b/doc/_config.yml index f2ddd7f58c..9dd7f40529 100644 --- a/doc/_config.yml +++ b/doc/_config.yml @@ -26,9 +26,10 @@ navbar: - user/getting_started/community.html.textile.liquid - Walkthough: - user/tutorials/wgs-tutorial.html.textile.liquid - - Run a workflow using Workbench: + - Using Workbench: - user/getting_started/workbench.html.textile.liquid - user/tutorials/tutorial-workflow-workbench.html.textile.liquid + - user/topics/workbench-migration.html.textile.liquid - Working at the Command Line: - user/getting_started/setup-cli.html.textile.liquid - user/reference/api-tokens.html.textile.liquid diff --git a/doc/_includes/_install_ruby_and_bundler.liquid b/doc/_includes/_install_ruby_and_bundler.liquid index ffaa1a1583..549e144634 100644 --- a/doc/_includes/_install_ruby_and_bundler.liquid +++ b/doc/_includes/_install_ruby_and_bundler.liquid @@ -4,7 +4,7 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} -Ruby 2.5 or newer is required. +Ruby 2.6 or newer is required. * "Option 1: Install from packages":#packages * "Option 2: Install with RVM":#rvm @@ -18,11 +18,13 @@ Future versions of Arvados may require a newer version of Ruby than is packaged h3. Centos 7 -The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install Ruby 2.5 or later. +The Ruby version shipped with Centos 7 is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer). h3. Debian and Ubuntu -Debian 10 (buster) and Ubuntu 18.04 (bionic) and later ship with Ruby 2.5 or newer, which is sufficient for Arvados. +Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer). + +Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados. 
# apt-get --no-install-recommends install ruby ruby-dev
@@ -48,10 +50,10 @@ h3. Install RVM, Ruby and Bundler 
# gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.5
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
-This command installs the latest Ruby 2.5.x release, as well as the @gem@ and @bundle@ commands. +This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands. To use Ruby installed from RVM, load it in an open shell like this: @@ -90,8 +92,8 @@ Build and install Ruby: 
mkdir -p ~/src
 cd ~/src
-curl -f http://cache.ruby-lang.org/pub/ruby/2.5/ruby-2.5.8.tar.gz | tar xz
-cd ruby-2.5.8
+curl -f https://cache.ruby-lang.org/pub/ruby/2.7/ruby-2.7.5.tar.gz | tar xz
+cd ruby-2.7.5
 ./configure --disable-install-rdoc
 make
 sudo make install
diff --git a/doc/_includes/_install_custom_certificates.liquid b/doc/_includes/_multi_host_install_custom_certificates.liquid
similarity index 57%
rename from doc/_includes/_install_custom_certificates.liquid
rename to doc/_includes/_multi_host_install_custom_certificates.liquid
index 4a4aff5cfb..b831aadcf9 100644
--- a/doc/_includes/_install_custom_certificates.liquid
+++ b/doc/_includes/_multi_host_install_custom_certificates.liquid
@@ -4,7 +4,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-If you plan to use custom certificates, please set the variable USE_LETSENCRYPT=no and copy your certificates to the directory specified with the variable @CUSTOM_CERTS_DIR@ (usually "./certs") in the remote directory where you copied the @provision.sh@ script. From this dir, the provision script will install the certificates required for the role you're installing.
+Copy your certificates to the directory specified with the variable @CUSTOM_CERTS_DIR@ in the remote directory where you copied the @provision.sh@ script. The provision script will find the certificates there.
 
 The script expects cert/key files with these basenames (matching the role except for keepweb, which is split in both download / collections):
 
@@ -17,10 +17,12 @@ The script expects cert/key files with these basenames (matching the role except
 * "collections"      # Part of keepweb
 * "keepproxy"
 
-Ie., for 'keepproxy', the script will look for
+E.g. for 'keepproxy', the script will look for
 
 
 
${CUSTOM_CERTS_DIR}/keepproxy.crt
 ${CUSTOM_CERTS_DIR}/keepproxy.key
 

 
+
+Make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
diff --git a/doc/admin/link-accounts.html.textile.liquid b/doc/admin/link-accounts.html.textile.liquid
index d0ac6a036a..6e880fdf66 100644
--- a/doc/admin/link-accounts.html.textile.liquid
+++ b/doc/admin/link-accounts.html.textile.liquid
@@ -11,6 +11,8 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 If a user needs to log in to Arvados with a upstream account or provider, they may end up with two Arvados user accounts.  If the user still has the ability to log in with the old account, they can use the "self-serve account linking":{{site.baseurl}}/user/topics/link-accounts.html feature of workbench.  However, if the user does not have the ability to log in with both upstream accounts, the admin can also link the accounts using the command line.
 
+bq. NOTE: self-serve account linking is currently not supported on LoginCluster federations and needs to be performed manually by the site admin.
+
 h3. Step 1: Determine user uuids
 
 User uuids can be determined by browsing workbench or using @arv user list@ at the command line.
diff --git a/doc/admin/spot-instances.html.textile.liquid b/doc/admin/spot-instances.html.textile.liquid
index 7ca57df0ab..3837f30d6d 100644
--- a/doc/admin/spot-instances.html.textile.liquid
+++ b/doc/admin/spot-instances.html.textile.liquid
@@ -16,13 +16,11 @@ Currently Arvados supports preemptible instances using AWS and Azure spot instan
 
 h2. Configuration
 
-First, ensure automatic selection of preemptible instances is not disabled in your configuration file (this is enabled by default, but can be disabled with @AlwaysUsePreemptibleInstances: false@), and add entries to @InstanceTypes@ that have @Preemptible: true@.  Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type.  The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower.  For example:
+Add entries to @InstanceTypes@ that have @Preemptible: true@.  Typically you want to add both preemptible and non-preemptible entries for each cloud provider VM type.  The @Price@ for preemptible instances is the maximum bid price, the actual price paid is dynamic and will likely be lower.  For example:
 
 
 Clusters:
-  ClusterID: 
-    Containers:
-      AlwaysUsePreemptibleInstances: true
+  ClusterID:
     InstanceTypes:
       m4.large:
         Preemptible: false
@@ -40,7 +38,18 @@ Clusters:
         Price: 0.1
 

 
-When @AlwaysUsePreemptibleInstances@ is enabled, child containers (workflow steps) will automatically be made preemptible.  Note that because preempting the workflow runner would cancel the entire workflow, the workflow runner runs in a reserved (non-preemptible) instance.
+Next, you can choose to enable automatic use of preemptible instances:
+
+
+    Containers:
+      AlwaysUsePreemptibleInstances: true
+

+
+If @AlwaysUsePreemptibleInstances@ is "true", child containers (workflow steps) will always select preemptible instances, regardless of user option.
+
+If @AlwaysUsePreemptibleInstances@ is "false" (the default) or unspecified, preemptible instance are "used when requested by the user.":{{site.baseurl}}/user/cwl/cwl-run-options.html#preemptible
+
+Note that regardless of the value of @AlwaysUsePreemptibleInstances@, the top level workflow runner container always runs in a reserved (non-preemptible) instance, to avoid situations where the workflow runner is killed requiring the entire to be restarted.
 
 No additional configuration is required, "arvados-dispatch-cloud":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html will now start preemptible instances where appropriate.
 
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 943bc3e0ee..abaa190c8c 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -28,10 +28,20 @@ TODO: extract this information based on git commit messages and generate changel
 

 
 
-h2(#main). development main (as of 2022-02-10)
+h2(#main). development main (as of 2022-03-08)
 
 "previous: Upgrading to 2.3.0":#v2_3_0
 
+h3. Ubuntu 18.04 Arvados Python packages now depend on python-3.8
+
+Ubuntu 18.04 ships with Python 3.6 as the default version of Python 3. Ubuntu also ships a version of Python 3.8, and the Arvados Python packages (@python3-arvados-cwl-runner@, @python3-arvados-fuse@, @python3-arvados-python-client@, @python3-arvados-user-activity@ and @python3-crunchstat-summary@) now depend on the @python-3.8@ system package.
+
+This means that they are now installed under @/usr/share/python3.8@ (before, the path was @/usr/share/python3@). If you rely on the @python3@ executable from the packages (e.g. to load a virtualenv), you may need to update the path to that executable.
+
+h3. Minimum supported Ruby version is now 2.6
+
+The minimum supported Ruby version is now 2.6.  If you are running Arvados on Debian 10 or Ubuntu 18.04, you may need to switch to using RVM or upgrade your OS.  See "Install Ruby and Bundler":../install/ruby.html for more information.
+
 h3. Anonymous token changes
 
 The anonymous token configured in @Users.AnonymousUserToken@ must now be 32 characters or longer. This was already the suggestion in the documentation, now it is enforced. The @script/get_anonymous_user_token.rb@ script that was needed to register the anonymous user token in the database has been removed. Registration of the anonymous token is no longer necessary. If the anonymous token in @config.yml@ is specified as a full V2 token, that will now generate a warning - it should be updated to list just the secret (i.e. the part after the last forward slash).
diff --git a/doc/api/index.html.textile.liquid b/doc/api/index.html.textile.liquid
index 8586a166d3..3d69d02ea9 100644
--- a/doc/api/index.html.textile.liquid
+++ b/doc/api/index.html.textile.liquid
@@ -20,6 +20,10 @@ h2. Exported configuration
 
 The Controller exposes a subset of the cluster's configuration and makes it available to clients in JSON format. This public config includes valuable information like several service's URLs, timeout settings, etc. and it is available at @/arvados/v1/config@, for example @https://{{ site.arvados_api_host }}/arvados/v1/config@. The new Workbench is one example of a client using this information, as it's a client-side application and doesn't have access to the cluster's config file.
 
+h2. Exported vocabulary definition
+
+When configured, the Controller also exports the "metadata vocabulary definition":{{site.baseurl}}/admin/metadata-vocabulary.html in JSON format. This functionality is useful for clients like Workbench2 and the Python SDK to provide "identifier to human-readable labels" translations facilities for reading and writing objects on the system. This is available at @/arvados/v1/vocabulary@, for example @https://{{ site.arvados_api_host }}/arvados/v1/vocabulary@.
+
 h2. Workbench examples
 
 Many Arvados Workbench pages, under the *Advanced* tab, provide examples of API and SDK use for accessing the current resource .
diff --git a/doc/api/methods/collections.html.textile.liquid b/doc/api/methods/collections.html.textile.liquid
index 01efda2b0c..5ff8d529f8 100644
--- a/doc/api/methods/collections.html.textile.liquid
+++ b/doc/api/methods/collections.html.textile.liquid
@@ -47,7 +47,7 @@ table(table table-bordered table-condensed).
 
 h3. Conditions of creating a Collection
 
-The @portable_data_hash@ and @manifest_text@ attributes must be provided when creating a Collection. The cryptographic digest of the supplied @manifest_text@ must match the supplied @portable_data_hash@.
+If a new @portable_data_hash@ is specified when creating or updating a Collection, it must match the cryptographic digest of the supplied @manifest_text@.
 
 h3. Side effects of creating a Collection
 
@@ -72,6 +72,9 @@ Arguments:
 table(table table-bordered table-condensed).
 |_. Argument |_. Type |_. Description |_. Location |_. Example |
 |collection|object||query||
+|replace_files|object|Initialize files and directories using content from other collections|query||
+
+The new collection's content can be initialized by providing a @manifest_text@ key in the provided @collection@ object, or by using the @replace_files@ option (see "replace_files":#replace_files below).
 
 h3. delete
 
@@ -116,6 +119,9 @@ table(table table-bordered table-condensed).
 |_. Argument |_. Type |_. Description |_. Location |_. Example |
 {background:#ccffcc}.|uuid|string|The UUID of the Collection in question.|path||
 |collection|object||query||
+|replace_files|object|Delete and replace files and directories using content from other collections|query||
+
+The collection's content can be updated by providing a @manifest_text@ key in the provided @collection@ object, or by using the @replace_files@ option (see "replace_files":#replace_files below).
 
 h3. untrash
 
@@ -160,3 +166,56 @@ Arguments:
 table(table table-bordered table-condensed).
 |_. Argument |_. Type |_. Description |_. Location |_. Example |
 {background:#ccffcc}.|uuid|string|The UUID of the Collection to get usage.|path||
+
+h2(#replace_files). Using "replace_files" to create/update collections
+
+The @replace_files@ option can be used with the @create@ and @update@ APIs to efficiently copy individual files and directory trees from other collections, and copy/rename/delete items within an existing collection, without transferring any file data.
+
+@replace_files@ keys indicate target paths in the new collection, and values specify sources that should be copied to the target paths.
+* Each target path must be an absolute canonical path beginning with @/@. It must not contain @.@ or @..@ components, consecutive @/@ characters, or a trailing @/@ after the final component.
+* Each source must be either an empty string (signifying that the target path is to be deleted), or @PDH/path@ where @PDH@ is the portable data hash of a collection on the cluster and @/path@ is a file or directory in that collection.
+* In an @update@ request, sources may reference the current portable data hash of the collection being updated.
+
+Example: delete @foo.txt@ from a collection
+
+
+"replace_files": {
+  "/foo.txt": ""
+}
+

+
+Example: rename @foo.txt@ to @bar.txt@ in a collection with portable data hash @fa7aeb5140e2848d39b416daeef4ffc5+45@
+
+
+"replace_files": {
+  "/foo.txt": "",
+  "/bar.txt": "fa7aeb5140e2848d39b416daeef4ffc5+45/foo.txt"
+}
+

+
+Example: delete current contents, then add content from multiple collections
+
+
+"replace_files": {
+  "/": "",
+  "/copy of collection 1": "1f4b0bc7583c2a7f9102c395f4ffc5e3+45/",
+  "/copy of collection 2": "ea10d51bcf88862dbcc36eb292017dfd+45/"
+}
+

+
+Example: replace entire collection with a copy of a subdirectory from another collection
+
+
+"replace_files": {
+  "/": "1f4b0bc7583c2a7f9102c395f4ffc5e3+45/subdir"
+}
+

+
+A target path with a non-empty source cannot be the ancestor of another target path in the same request. For example, the following request is invalid:
+
+
+"replace_files": {
+  "/foo": "fa7aeb5140e2848d39b416daeef4ffc5+45/",
+  "/foo/this_will_return_an_error": ""
+}
+

diff --git a/doc/images/switch-to-wb1.png b/doc/images/switch-to-wb1.png
new file mode 100644
index 0000000000..3787e31535
Binary files /dev/null and b/doc/images/switch-to-wb1.png differ
diff --git a/doc/images/switch-to-wb2.png b/doc/images/switch-to-wb2.png
new file mode 100644
index 0000000000..177090b329
Binary files /dev/null and b/doc/images/switch-to-wb2.png differ
diff --git a/doc/images/wb2-example.png b/doc/images/wb2-example.png
new file mode 100644
index 0000000000..7bdea9e78b
Binary files /dev/null and b/doc/images/wb2-example.png differ
diff --git a/doc/install/install-manual-prerequisites.html.textile.liquid b/doc/install/install-manual-prerequisites.html.textile.liquid
index 360cfbabdd..a9a91ab3cb 100644
--- a/doc/install/install-manual-prerequisites.html.textile.liquid
+++ b/doc/install/install-manual-prerequisites.html.textile.liquid
@@ -27,7 +27,7 @@ The Arvados storage subsystem is called "keep".  The compute subsystem is called
 h2(#supportedlinux). Supported GNU/Linux distributions
 
 table(table table-bordered table-condensed).
-|_. Distribution|_. State|_. Last supported version|
+|_. Distribution|_. State|_. Last supported Arvados version|
 |CentOS 7|Supported|Latest|
 |Debian 11 ("bullseye")|Supported|Latest|
 |Debian 10 ("buster")|Supported|Latest|
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
index 10f2e32ef1..1778338f53 100644
--- a/doc/install/salt-multi-host.html.textile.liquid
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -47,6 +47,7 @@ We suggest distributing the Arvados components in the following way, creating at
 ## arvados controller
 ## arvados websocket
 ## arvados cloud dispatcher
+## arvados keepbalance
 # WORKBENCH node:
 ## arvados workbench
 ## arvados workbench2
@@ -98,7 +99,9 @@ Edit the variables in the local.params file. Pay attention to the *_IN
 
 The multi_host example includes Let's Encrypt salt code to automatically request and install the certificates for the public-facing hosts (API/controller, Workbench, Keepproxy/Keepweb) using AWS' Route53.
 
-{% include 'install_custom_certificates' %}
+{% include 'multi_host_install_custom_certificates' %}
+
+If you want to use valid certificates provided by Let's Encrypt, set the variable SSL_MODE=lets-encrypt and make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
 
 h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
 
@@ -148,7 +151,7 @@ ssh user@host sudo ./provision.sh --config local.params --roles database
 h4. API
 
 
scp -r provision.sh local* user@host:
-ssh user@host sudo ./provision.sh --config local.params --roles api,controller,websocket,dispatcher
+ssh user@host sudo ./provision.sh --config local.params --roles api,controller,websocket,dispatcher,keepbalance
 

 
 
diff --git a/doc/install/salt-single-host.html.textile.liquid b/doc/install/salt-single-host.html.textile.liquid
index 0f06412f91..106fab9bd4 100644
--- a/doc/install/salt-single-host.html.textile.liquid
+++ b/doc/install/salt-single-host.html.textile.liquid
@@ -9,109 +9,151 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-# "Single host install using the provision.sh script":#single_host
-# "Choose the desired configuration":#choose_configuration
-## "Single host / single hostname":#single_host_single_hostnames
-## "Single host / multiple hostnames (Alternative configuration)":#single_host_multiple_hostnames
-## "Further customization of the installation (modifying the salt pillars and states)":#further_customization
+# "Limitations of the single host install":#limitations
+# "Prerequisites":#prerequisites
+# "Download the installer":#single_host
+# "Choose the SSL configuration":#certificates
+## "Using a self-signed certificate":#self-signed
+## "Using a Let's Encrypt certificate":#lets-encrypt
+## "Bring your own certificate":#bring-your-own
+# "Further customization of the installation (modifying the salt pillars and states)":#further_customization
 # "Run the provision.sh script":#run_provision_script
-# "Final configuration steps":#final_steps
-## "Install the CA root certificate (required in both alternatives)":#ca_root_certificate
-## "DNS configuration (single host / multiple hostnames)":#single_host_multiple_hostnames_dns_configuration
+# "Install the CA root certificate":#ca_root_certificate
 # "Initial user and login":#initial_user
 # "Test the installed cluster running a simple workflow":#test_install
 # "After the installation":#post_install
 
-h2(#single_host). Single host install using the provision.sh script
+h2(#limitations). Limitations of the single host install
 
-NOTE: The single host installation is not recommended for production use.
+NOTE: The single host installation is a good choice for evaluating Arvados, but it is not recommended for production use.
 
-{% include 'branchname' %}
+Using the default configuration, this installation method has a number of limitations:
 
-This is a package-based installation method. Start with the @provision.sh@ script which is available by cloning the @{{ branchname }}@ branch from "https://git.arvados.org/arvados.git":https://git.arvados.org/arvados.git .  The @provision.sh@ script and its supporting files can be found in the "arvados/tools/salt-install":https://git.arvados.org/arvados.git/tree/refs/heads/{{ branchname }}:/tools/salt-install directory in the Arvados git repository.
+* all services run on the same machine, and they will compete for resources. This includes any compute jobs.
+* it uses the local machine disk for Keep storage (under the @/tmp@ directory). There may not be a lot of space available.
+* it installs the @crunch-dispatch-local@ dispatcher, which can run just eight concurrent CWL jobs. These jobs will be executed on the same machine that runs all the Arvados services and may well starve them of resources.
 
-This procedure will install all the main Arvados components to get you up and running in a single host. The whole installation procedure takes somewhere between 15 to 60 minutes, depending on the host resources and its network bandwidth. As a reference, on a virtual machine with 1 core and 1 GB RAM, it takes ~25 minutes to do the initial install.
+It is possible to start with the single host installation method and modify the Arvados configuration file later to address these limitations. E.g. switch to a "different storage volume setup":{{site.baseurl}}/install/configure-s3-object-storage.html for Keep, and switch to "the cloud dispatcher":{{site.baseurl}}/install/crunch2-cloud/install-dispatch-cloud.html to provision compute resources dynamically.
 
-The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located at "arvados-formula":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script.  The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+h2(#prerequisites). Prerequisites and planning
 
-After setting up a few variables in a config file (next step), you'll be ready to run it and get Arvados deployed.
+Prerequisites:
 
-h2(#choose_configuration). Choose the desired configuration
+* git
+* a dedicated (virtual) machine for your Arvados server with at least 2 cores and 8 GiB of RAM, running a "supported Arvados distribution":{{site.baseurl}}/install/install-manual-prerequisites.html#supportedlinux
+* a DNS hostname that resolves to the IP address of your Arvados server
+* ports 443, 8800-8805 need to be reachable from your client (configurable in @local.params@, see below)
+* port 80 needs to be reachable from everywhere on the internet (only when using "Let's Encrypt":#lets-encrypt)
+* an SSL certificate matching the hostname in use (only when using "bring your own certificate":#bring-your-own)
 
-For documentation's sake, we will use the cluster name arva2 and the domain arv.local. If you don't change them as required in the next steps, installation won't proceed.
+h2(#single_host). Download the installer
 
-Arvados' single host installation can be done in two fashions:
+{% include 'branchname' %}
 
-* Using a single hostname, assigning a different port (other than 443) for each user-facing service: This choice is easier to setup, but the user will need to know the port/s for the different services she wants to connect to.
-* Using multiple hostnames on the same IP: this setup involves a few extra steps but each service will have a meaningful hostname so it will make easier to access them later.
+This procedure will install all the main Arvados components to get you up and running in a single host.
 
-Once you decide which of these choices you prefer, copy one the two example configuration files and directory, and edit them to suit your needs.
+This is a package-based installation method, however the installation script is currently distributed in source form via @git@:
+
+
+
git clone https://git.arvados.org/arvados.git
+git checkout {{ branchname }}
+cd arvados/tools/salt-install
+

+
+
+The @provision.sh@ script will help you deploy Arvados by preparing your environment to be able to run the installer, then running it. The actual installer is located in the "arvados-formula git repository":https://git.arvados.org/arvados-formula.git/tree/refs/heads/{{ branchname }} and will be cloned during the running of the @provision.sh@ script.  The installer is built using "Saltstack":https://saltproject.io/ and @provision.sh@ performs the install using master-less mode.
+
+First, copy the configuration files:
 
-h3(#single_host_single_hostnames). Single host / single hostname
 
 
cp local.params.example.single_host_single_hostname local.params
 cp -r config_examples/single_host/single_hostname local_config_dir
 

 
 
-Edit the variables in the local.params file. Pay attention to the *_PORT, *_TOKEN and *KEY variables.
+Edit the variables in the local.params file. Pay attention to the *_PORT, *_TOKEN and *KEY variables. The *SSL_MODE* variable is discussed in the next section.
+
+h2(#certificates). Choose the SSL configuration (SSL_MODE)
+
+Arvados requires an SSL certificate to work correctly. This installer supports these options:
 
-The single_host examples use self-signed SSL certificates, which are deployed using the same mechanism used to deploy custom certificates.
+* @self-signed@: let the installer create a self-signed certificate
+* @lets-encrypt@: automatically obtain and install an SSL certificate for your hostname
+* @bring-your-own@: supply your own certificate in the `certs` directory
 
-{% include 'install_custom_certificates' %}
+h3(#self-signed). Using a self-signed certificate
 
-If you want to use valid certificates provided by Let's Encrypt, please set the variable USE_LETSENCRYPT=yes and make sure that all the FQDNs that you will use for the public-facing applications (API/controller, Workbench, Keepproxy/Keepweb) are reachable.
+In the default configuration, this installer uses self-signed certificate(s):
 
-h3(#single_host_multiple_hostnames). Single host / multiple hostnames (Alternative configuration)
 
-
cp local.params.example.single_host_multiple_hostnames local.params
-cp -r config_examples/single_host/multiple_hostnames local_config_dir
+
SSL_MODE="self-signed"
 

 
 
-Edit the variables in the local.params file.
+When connecting to the Arvados web interface for the first time, you will need to accept the self-signed certificate as trusted to bypass the browser warnings.
 
-h3(#further_customization). Further customization of the installation (modifying the salt pillars and states)
+h3(#lets-encrypt). Using a Let's Encrypt certificate
 
-If you want or need further customization, you can edit the Saltstack pillars and states files. Pay particular attention to the pillars/arvados.sls one. Any extra state file you add under local_config_dir/states will be added to the salt run and applied to the host.
+To automatically get a valid certificate via Let's Encrypt, change the configuration like this:
 
-h2(#run_provision_script). Run the provision.sh script
+
+
SSL_MODE="lets-encrypt"
+

+
 
-When you finished customizing the configuration, you are ready to copy the files to the host (if needed) and run the @provision.sh@ script:
+The hostname for your Arvados cluster must be defined in @HOSTNAME_EXT@ and resolve to the public IP address of your Arvados instance, so that Let's Encrypt can validate the domainname ownership and issue the certificate.
+
+When using AWS, EC2 instances can have a default hostname that ends with amazonaws.com. Let's Encrypt has a blacklist of domain names for which it will not issue certificates, and that blacklist includes the amazonaws.com domain, which means the default hostname can not be used to get a certificate from Let's Encrypt.
+
+h3(#bring-your-own). Bring your own certificate
+
+To supply your own certificate, change the configuration like this:
 
 
-
scp -r provision.sh local* tests user@host:
-# if you use custom certificates (not Let's Encrypt), make sure to copy those too:
-# scp -r certs user@host:
-ssh user@host sudo ./provision.sh
+
SSL_MODE="bring-your-own"
+CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
 

 
 
-or, if you saved the @local.params@ in another directory or with some other name
+Copy your certificate files to the directory specified with the variable @CUSTOM_CERTS_DIR@. The provision script will find it there. The certificate and its key need to be copied to a file named after @HOSTNAME_EXT@. For example, if @HOSTNAME_EXT@ is defined as @my-arvados.example.net@, the script will look for
 
 
-
scp -r provision.sh local* tests user@host:
-ssh user@host sudo ./provision.sh -c /path/to/your/local.params.file
+
${CUSTOM_CERTS_DIR}/my-arvados.example.net.crt
+${CUSTOM_CERTS_DIR}/my-arvados.example.net.key
 

 
 
-and wait for it to finish.
+All certificate files will be used by nginx. You may need to include intermediate certificates in your certificate file. See "the nginx documentation":http://nginx.org/en/docs/http/configuring_https_servers.html#chains for more details.
+
+h2(#further_customization). Further customization of the installation (modifying the salt pillars and states)
+
+If you want or need further customization, you can edit the Saltstack pillars and states files. Pay particular attention to the pillars/arvados.sls one. Any extra state file you add under local_config_dir/states will be added to the salt run and applied to the host.
+
+h2(#run_provision_script). Run the provision.sh script
 
-If everything goes OK, you'll get some final lines stating something like:
+When you finished customizing the configuration, you are ready to copy the files to the host (if needed) and run the @provision.sh@ script:
 
 
-
arvados: Succeeded: 109 (changed=9)
-arvados: Failed:      0
+
scp -r provision.sh local* tests user@host:
+# if you have set SSL_MODE to "bring-your-own", make sure to also copy the certificate files:
+# scp -r certs user@host:
+ssh user@host sudo ./provision.sh
 

 
 
-h2(#final_steps). Final configuration steps
+and wait for it to finish. The script will need 5 to 10 minutes to install and configure everything.
+
+If everything goes OK, you'll get final output that looks similar to this:
 
-Once the deployment went OK, you'll need to perform a few extra steps in your local browser/host to access the cluster.
+
+
arvados: Succeeded: 151 (changed=36)
+arvados: Failed:      0
+

+
 
-h3(#ca_root_certificate). Install the CA root certificate (required in both alternatives)
+h2(#ca_root_certificate). Install the CA root certificate (SSL_MODE=self-signed only)
 
-Arvados uses SSL to encrypt communications. Its UI uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
+Arvados uses SSL to encrypt communications. The web interface uses AJAX which will silently fail if the certificate is not valid or signed by an unknown Certification Authority.
 
 For this reason, the @arvados-formula@ has a helper state to create a root certificate to authorize Arvados services. The @provision.sh@ script will leave a copy of the generated CA's certificate (@arvados-snakeoil-ca.pem@) in the script's directory so you can add it to your workstation.
 
@@ -142,39 +184,13 @@ To access your Arvados instance using command line clients (such as arv-get and
 

 
 
-h3(#single_host_multiple_hostnames_dns_configuration). DNS configuration (single host / multiple hostnames)
-
-When using multiple hostnames, after the setup is done, you need to set up your DNS to be able to access the cluster.
-
-If you don't have access to the domain's DNS to add the required entries, the simplest way to do it is to edit your @/etc/hosts@ file (as root):
-
-
-
export CLUSTER="arva2"
-export DOMAIN="arv.local"
-export HOST_IP="127.0.0.2"    # This is valid either if installing in your computer directly
-                              # or in a Vagrant VM. If you're installing it on a remote host
-                              # just change the IP to match that of the host.
-echo "${HOST_IP} api keep keep0 collections download ws workbench workbench2 ${CLUSTER}.${DOMAIN} api.${CLUSTER}.${DOMAIN} keep.${CLUSTER}.${DOMAIN} keep0.${CLUSTER}.${DOMAIN} collections.${CLUSTER}.${DOMAIN} download.${CLUSTER}.${DOMAIN} ws.${CLUSTER}.${DOMAIN} workbench.${CLUSTER}.${DOMAIN} workbench2.${CLUSTER}.${DOMAIN}" >> /etc/hosts
-

-
-
 h2(#initial_user). Initial user and login
 
-At this point you should be able to log into the Arvados cluster. The initial URL will be:
-
-* https://workbench.arva2.arv.local
-
-or, in general, the url format will be:
+At this point you should be able to log on to your new Arvados cluster. The workbench URL will be
 
-* https://workbench.@.@
+* https://@HOSTNAME_EXT@
 
-By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster.
-
-Assuming you didn't change these values in the @local.params@ file, the initial credentials are:
-
-* User: 'admin'
-* Password: 'password'
-* Email: 'admin@arva2.arv.local'
+By default, the provision script creates an initial user for testing purposes. This user is configured as administrator of the newly created cluster. The username, password and e-mail address for the initial user are configured in the @local.params@ file. Log in with the e-mail address and password.
 
 h2(#test_install). Test the installed cluster running a simple workflow
 
@@ -197,15 +213,6 @@ Arvados project uuid is 'arva2-j7d0g-0prd8cjlk6kfl7y'
  "owner_uuid":"arva2-tpzed-000000000000000",
  ...
 }
-Uploading arvados/jobs' docker image to the project
-2.1.1: Pulling from arvados/jobs
-8559a31e96f4: Pulling fs layer
-...
-Status: Downloaded newer image for arvados/jobs:2.1.1
-docker.io/arvados/jobs:2.1.1
-2020-11-23 21:43:39 arvados.arv_put[32678] INFO: Creating new cache file at /home/vagrant/.cache/arvados/arv-put/c59256eda1829281424c80f588c7cc4d
-2020-11-23 21:43:46 arvados.arv_put[32678] INFO: Collection saved as 'Docker image arvados jobs:2.1.1 sha256:0dd50'
-arva2-4zz18-1u5pvbld7cvxuy2
 Creating initial user ('admin')
 Setting up user ('admin')
 {
diff --git a/doc/install/salt.html.textile.liquid b/doc/install/salt.html.textile.liquid
index 8db0ac15e4..29a6eacf3b 100644
--- a/doc/install/salt.html.textile.liquid
+++ b/doc/install/salt.html.textile.liquid
@@ -31,8 +31,6 @@ You don't need to be running a Saltstack infrastructure to install Arvados: we w
 
 This is a package-based installation method.
 
-
-
 h2(#provisioning_arvados). Provisioning Arvados with Saltstack
 
 The "tools/salt-install":https://git.arvados.org/arvados.git/tree/{{ branchname }}:/tools/salt-install directory in the Arvados git repository contains a script that you can run in the node/s where you want to install Arvados' components (the @provision.sh@ script) and a few configuration examples for different setups, that you can use to customize your installation.
diff --git a/doc/sdk/python/cookbook.html.textile.liquid b/doc/sdk/python/cookbook.html.textile.liquid
index eda6563d7a..f3186ebbb6 100644
--- a/doc/sdk/python/cookbook.html.textile.liquid
+++ b/doc/sdk/python/cookbook.html.textile.liquid
@@ -298,3 +298,37 @@ api = arvados.api()
 for c in arvados.util.keyset_list_all(api.collections().list, filters=[["name", "like", "%sample123%"]]):
     print("got collection " + c["uuid"])
 {% endcodeblock %}
+
+h2. Querying the vocabulary definition
+
+The Python SDK provides facilities to interact with the "active metadata vocabulary":{{ site.baseurl }}/admin/metadata-vocabulary.html in the system. The developer can do key and value lookups in a case-insensitive manner:
+
+{% codeblock as python %}
+from arvados import api, vocabulary
+voc = vocabulary.load_vocabulary(api('v1'))
+
+[k.identifier for k in set(voc.key_aliases.values())]
+# Example output: ['IDTAGCOLORS', 'IDTAGFRUITS', 'IDTAGCOMMENT', 'IDTAGIMPORTANCES', 'IDTAGCATEGORIES', 'IDTAGSIZES', 'IDTAGANIMALS']
+voc['IDTAGSIZES'].preferred_label
+# Example output: 'Size'
+[v.preferred_label for v in set(voc['size'].value_aliases.values())]
+# Example output: ['S', 'M', 'L', 'XL', 'XS']
+voc['size']['s'].aliases
+# Example output: ['S', 'small']
+voc['size']['Small'].identifier
+# Example output: 'IDVALSIZES2'
+{% endcodeblock %}
+
+h2. Translating between vocabulary identifiers and labels
+
+Client software might need to present properties to the user in a human-readable form or take input from the user without requiring them to remember identifiers. For these cases, there're a couple of conversion methods that take a dictionary as input like this:
+
+{% codeblock as python %}
+from arvados import api, vocabulary
+voc = vocabulary.load_vocabulary(api('v1'))
+
+voc.convert_to_labels({'IDTAGIMPORTANCES': 'IDVALIMPORTANCES1'})
+# Example output: {'Importance': 'Critical'}
+voc.convert_to_identifiers({'creature': 'elephant'})
+# Example output: {'IDTAGANIMALS': 'IDVALANIMALS3'}
+{% endcodeblock %}
\ No newline at end of file
diff --git a/doc/sdk/python/sdk-python.html.textile.liquid b/doc/sdk/python/sdk-python.html.textile.liquid
index 435f70e7bf..56f0328042 100644
--- a/doc/sdk/python/sdk-python.html.textile.liquid
+++ b/doc/sdk/python/sdk-python.html.textile.liquid
@@ -65,6 +65,10 @@ Type "help", "copyright", "credits" or "license" for more information.
 
 If you installed from a distribution package (option 2): the package includes a virtualenv, which means the correct Python environment needs to be loaded before the Arvados SDK can be imported. This can be done by activating the virtualenv first:
 
+{% include 'notebox_begin_warning' %}
+If you are on Ubuntu 18.04, please note that the Arvados packages that use Python depend on the python-3.8 package. This means they are installed under @/usr/share/python3.8@, not @/usr/share/python3@. You will need to update the commands below accordingly.
+{% include 'notebox_end' %}
+
 
 
~$ source /usr/share/python3/dist/python3-arvados-python-client/bin/activate
 (python-arvados-python-client) ~$ python
diff --git a/doc/user/cwl/cwl-extensions.html.textile.liquid b/doc/user/cwl/cwl-extensions.html.textile.liquid
index dd78e989fd..d6148d7eee 100644
--- a/doc/user/cwl/cwl-extensions.html.textile.liquid
+++ b/doc/user/cwl/cwl-extensions.html.textile.liquid
@@ -63,6 +63,9 @@ hints:
     cudaComputeCapabilityMin: "9.0"
     deviceCountMin: 1
     deviceCountMax: 1
+
+  arv:UsePreemptible:
+    usePreemptible: true
 {% endcodeblock %}
 
 h2(#RunInSingleContainer). arv:RunInSingleContainer
@@ -164,6 +167,14 @@ table(table table-bordered table-condensed).
 |deviceCountMin|integer|Minimum number of GPU devices to allocate on a single node. Required.|
 |deviceCountMax|integer|Maximum number of GPU devices to allocate on a single node. Optional.  If not specified, same as @minDeviceCount@.|
 
+h2(#UsePreemptible). arv:UsePreemptible
+
+Specify whether a workflow step should request preemptible (e.g. AWS Spot market) instances.  Such instances are generally cheaper, but can be taken back by the cloud provider at any time (preempted) causing the step to fail.  When this happens, Arvados will automatically re-try the step, up to the configuration value of @Containers.MaxRetryAttempts@ (default 3) times.
+
+table(table table-bordered table-condensed).
+|_. Field |_. Type |_. Description |
+|usePreemptible|boolean|Required, true to opt-in to using preemptible instances, false to opt-out.|
+
 h2. arv:dockerCollectionPDH
 
 This is an optional extension field appearing on the standard @DockerRequirement@.  It specifies the portable data hash of the Arvados collection containing the Docker image.  If present, it takes precedence over @dockerPull@ or @dockerImageId@.
diff --git a/doc/user/cwl/cwl-run-options.html.textile.liquid b/doc/user/cwl/cwl-run-options.html.textile.liquid
index d331dad871..94e46ae1bc 100644
--- a/doc/user/cwl/cwl-run-options.html.textile.liquid
+++ b/doc/user/cwl/cwl-run-options.html.textile.liquid
@@ -63,6 +63,9 @@ table(table table-bordered table-condensed).
 |==--priority== PRIORITY|Workflow priority (range 1..1000, higher has precedence over lower)|
 |==--thread-count== THREAD_COUNT|Number of threads to use for container submit and output collection.|
 |==--http-timeout== HTTP_TIMEOUT|API request timeout in seconds. Default is 300 seconds (5 minutes).|
+|==--enable-preemptible==|Use preemptible instances. Control individual steps with "arv:UsePreemptible":cwl-extensions.html#UsePreemptible hint.|
+|==--disable-preemptible==|Don't use preemptible instances.|
+|==--skip-schemas==|Skip loading of extension schemas (the $schemas section).|
 |==--trash-intermediate==|Immediately trash intermediate outputs on workflow success.|
 |==--no-trash-intermediate==|Do not trash intermediate outputs (default).|
 
@@ -143,3 +146,19 @@ Using @--intermediate-output-ttl@ without @--trash-intermediate@ means that inte
 h3(#federation). Run workflow on a remote federated cluster
 
 By default, the workflow runner will run on the local (home) cluster.  Using @--submit-runner-cluster@ you can specify that the runner should be submitted to a remote federated cluster.  When doing this, @--project-uuid@ should specify a project on that cluster.  Steps making up the workflow will be submitted to the remote federated cluster by default, but the behavior of @arv:ClusterTarget@ is unchanged.  Note: when using this option, any resources that need to be uploaded in order to run the workflow (such as files or Docker images) will be uploaded to the local (home) cluster, and streamed to the federated cluster on demand.
+
+h3(#preemptible). Using preemptible (spot) instances
+
+Preemptible instances typically offer lower cost computation with a tradeoff of lower service guarantees.  If a compute node is preempted, Arvados will restart the computation on a new instance.
+
+If the sitewide configuration @Containers.AlwaysUsePreemptibleInstances@ is true, workflow steps will always select preemptible instances, regardless of user option.
+
+If @Containers.AlwaysUsePreemptibleInstances@ is false, you can request preemptible instances for a specific run with the @arvados-cwl-runner --enable-preemptible@ option.
+
+Within the workflow, you can control whether individual steps should be preemptible with the "arv:UsePreemptible":cwl-extensions.html#UsePreemptible hint.
+
+If a workflow requests preemptible instances with "arv:UsePreemptible":cwl-extensions.html#UsePreemptible , but you _do not_ want to use preemptible instances, you can override it for a specific run with the @arvados-cwl-runner --disable-preemptible@ option.
+
+h3(#gpu). Use CUDA GPU instances
+
+See "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement .
diff --git a/doc/user/cwl/cwl-style.html.textile.liquid b/doc/user/cwl/cwl-style.html.textile.liquid
index 853ed3b3e2..303ae37e9e 100644
--- a/doc/user/cwl/cwl-style.html.textile.liquid
+++ b/doc/user/cwl/cwl-style.html.textile.liquid
@@ -13,7 +13,15 @@ h2(#performance). Performance
 
 To get the best perfomance from your workflows, be aware of the following Arvados features, behaviors, and best practices.
 
-Does your application support NVIDIA GPU acceleration?  Use "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement to request nodes with GPUs.
+h3. Does your application support NVIDIA GPU acceleration?
+
+Use "cwltool:CUDARequirement":cwl-extensions.html#CUDARequirement to request nodes with GPUs.
+
+h3. Trying to reduce costs?
+
+Try "using preemptible (spot) instances":cwl-run-options.html#preemptible .
+
+h3. You have a sequence of short-running steps
 
 If you have a sequence of short-running steps (less than 1-2 minutes each), use the Arvados extension "arv:RunInSingleContainer":cwl-extensions.html#RunInSingleContainer to avoid scheduling and data transfer overhead by running all the steps together in the same container on the same node.  To use this feature, @cwltool@ must be installed in the container image.  Example:
 
@@ -42,10 +50,16 @@ steps:
     run: subworkflow-with-short-steps.cwl
 {% endcodeblock %}
 
+h3. Avoid declaring @InlineJavascriptRequirement@ or @ShellCommandRequirement@
+
 Avoid declaring @InlineJavascriptRequirement@ or @ShellCommandRequirement@ unless you specifically need them.  Don't include them "just in case" because they change the default behavior and may add extra overhead.
 
+h3. Prefer text substitution to Javascript
+
 When combining a parameter value with a string, such as adding a filename extension, write @$(inputs.file.basename).ext@ instead of @$(inputs.file.basename + 'ext')@.  The first form is evaluated as a simple text substitution, the second form (using the @+@ operator) is evaluated as an arbitrary Javascript expression and requires that you declare @InlineJavascriptRequirement@.
 
+h3. Use @ExpressionTool@ to efficiently rearrange input files
+
 Use @ExpressionTool@ to efficiently rearrange input files between steps of a Workflow.  For example, the following expression accepts a directory containing files paired by @_R1_@ and @_R2_@ and produces an array of Directories containing each pair.
 
 {% codeblock as yaml %}
@@ -80,9 +94,13 @@ expression: |
   }
 {% endcodeblock %}
 
-Available compute nodes types vary over time and across different cloud providers, so try to limit the RAM requirement to what the program actually needs.  However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM
+h3. Limit RAM requests to what you really need
+
+Available compute nodes types vary over time and across different cloud providers, so it is important to limit the RAM requirement to what the program actually needs.  However, if you need to target a specific compute node type, see this discussion on "calculating RAM request and choosing instance type for containers.":{{site.baseurl}}/api/execution.html#RAM
 
-Instead of scattering separate steps, prefer to scatter over a subworkflow.
+h3. Avoid scattering by step by step
+
+Instead of a scatter step that feeds into another scatter step, prefer to scatter over a subworkflow.
 
 With the following pattern, @step1@ has to wait for all samples to complete before @step2@ can start computing on any samples.  This means a single long-running sample can prevent the rest of the workflow from moving on:
 
@@ -148,10 +166,16 @@ h2. Portability
 
 To write workflows that are easy to modify and portable across CWL runners (in the event you need to share your workflow with others), there are several best practices to follow:
 
+h3. Always provide @DockerRequirement@
+
 Workflows should always provide @DockerRequirement@ in the @hints@ or @requirements@ section.
 
+h3. Build a reusable library of components
+
 Build a reusable library of components.  Share tool wrappers and subworkflows between projects.  Make use of and contribute to "community maintained workflows and tools":https://github.com/common-workflow-library and tool registries such as "Dockstore":http://dockstore.org .
 
+h3. Supply scripts as input parameters
+
 CommandLineTools wrapping custom scripts should represent the script as an input parameter with the script file as a default value.  Use @secondaryFiles@ for scripts that consist of multiple files.  For example:
 
 {% codeblock as yaml %}
@@ -180,22 +204,21 @@ outputs:
       glob: "*.fastq"
 {% endcodeblock %}
 
+h3. Getting the temporary and output directories
+
 You can get the designated temporary directory using @$(runtime.tmpdir)@ in your CWL file, or from the @$TMPDIR@ environment variable in your script.
 
 Similarly, you can get the designated output directory using $(runtime.outdir), or from the @HOME@ environment variable in your script.
 
-Avoid specifying resource requirements in CommandLineTool.  Prefer to specify them in the workflow.  You can provide a default resource requirement in the top level @hints@ section, and individual steps can override it with their own resource requirement.
+h3. Specifying @ResourceRequirement@
+
+Avoid specifying resources in the @requirements@ section of a @CommandLineTool@, put it in the @hints@ section instead.  This enables you to override the tool resource hint with a workflow step level requirement:
 
 {% codeblock as yaml %}
 cwlVersion: v1.0
 class: Workflow
 inputs:
   inp: File
-hints:
-  ResourceRequirement:
-    ramMin: 1000
-    coresMin: 1
-    tmpdirMin: 45000
 steps:
   step1:
     in: {inp: inp}
@@ -205,7 +228,7 @@ steps:
     in: {inp: step1/inp}
     out: [out]
     run: tool2.cwl
-    hints:
+    requirements:
       ResourceRequirement:
         ramMin: 2000
         coresMin: 2
diff --git a/doc/user/getting_started/workbench.html.textile.liquid b/doc/user/getting_started/workbench.html.textile.liquid
index 644cf7d208..7091e31eae 100644
--- a/doc/user/getting_started/workbench.html.textile.liquid
+++ b/doc/user/getting_started/workbench.html.textile.liquid
@@ -10,9 +10,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
 {% include 'notebox_begin' %}
-This guide covers the classic Arvados Workbench web application, sometimes referred to as "Workbench 1".  There is also a new Workbench web application under development called "Workbench 2".  Sites which have both Workbench applications installed will have a dropdown menu option "Switch to Workbench 2" to switch between versions.
-
-This guide will be updated to cover "Workbench 2" in the future.
+This guide covers the classic Arvados Workbench web application, sometimes referred to as "Workbench 1".  There is also a new Workbench web application under development called "Workbench 2".  This guide will be updated to cover "Workbench 2" in the future.  See "Workbench 2 migration":{{site.baseurl}}/user/topics/workbench-migration.html for more information.
 {% include 'notebox_end' %}
 
 You can access the Arvados Workbench used in this guide using this link:
diff --git a/doc/user/topics/link-accounts.html.textile.liquid b/doc/user/topics/link-accounts.html.textile.liquid
index 3854bf6496..8cfb935f28 100644
--- a/doc/user/topics/link-accounts.html.textile.liquid
+++ b/doc/user/topics/link-accounts.html.textile.liquid
@@ -11,6 +11,8 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 This page describes how to link additional login accounts to the same Arvados account.  This can be used to migrate login accounts, for example, from one Google account to another.  It can also be used to migrate login providers, for example from LDAP to Google.  In order to do this, you must be able to log into both the "old" and "new" accounts.
 
+bq. NOTE: If you need to link your accounts on an Arvados cluster federation where user management is centralized, this feature may not be available. If that's the case, the federation admin can do the linking manually.
+
 h2. Link accounts
 
 Follow this process to link the "new" login to the "old" login.
diff --git a/doc/user/topics/workbench-migration.html.textile.liquid b/doc/user/topics/workbench-migration.html.textile.liquid
new file mode 100644
index 0000000000..9a36435eac
--- /dev/null
+++ b/doc/user/topics/workbench-migration.html.textile.liquid
@@ -0,0 +1,49 @@
+---
+layout: default
+navsection: userguide
+title: "Workbench 2 migration"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+Arvados is in the process of migrating from the classic web application, referred to as "Workbench 1", to a completely new web application, referred to as "Workbench 2".
+
+!{width: 90%}{{ site.baseurl }}/images/wb2-example.png!
+
+Workbench 2 is the new Workbench web application that will, over time, replace Workbench 1. Workbench 2 is being built based on user feedback, and it is approaching feature parity with Workbench 1.  Workbench 2 has a modern look and feel and offers several advanced features and performance enhancements.  Arvados clusters typically have both Workbench applications installed and have a dropdown menu option in the user menu to switch between versions.
+
+!{{ site.baseurl }}/images/switch-to-wb2.png!
+
+Workbench 2 is stable and recommended for general use, but still lacks some features available in the classic Workbench 1 application.  When necessary, you can easily switch back:
+
+!{{ site.baseurl }}/images/switch-to-wb1.png!
+
+Some major improvements of Workbench 2 include:
+
+h2. General
+
+* More responsive, only loads data needed for display
+* More familiar user interface, modeled on the file explorer of MacOS and Windows.
+* Advanced search capabilities
+
+h2. Project browsing
+
+* Expanded informational columns
+* Expanded filtering options
+* Right side informational panel providing details about selected item without navigating away from the project
+* Support for adding and querying user-supplied metadata properties on Projects
+
+h2. Collection browsing
+
+* Able to browse collections with millions of files
+* Support for adding and querying user-supplied metadata properties on Collections
+* Support for viewing past versions of a collection
+
+h2. User and Group management
+
+* Able to create user groups through the GUI
+* Able to add/view/remove members of user groups, and what permissions are shared with the group
+* Able to add/view/remove permissions shared with individual users
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 8da58a682d..1b75e13420 100644
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -26,8 +26,8 @@ RUN apt-get update -q
 RUN apt-get install -yq --no-install-recommends python3-arvados-cwl-runner=$cwl_runner_version
 
 # use the Python executable from the python-arvados-cwl-runner package
-RUN rm -f /usr/bin/python && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python
-RUN rm -f /usr/bin/python3 && ln -s /usr/share/python3/dist/python3-arvados-cwl-runner/bin/python /usr/bin/python3
+RUN PYTHON=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python|head -n1` && rm -f /usr/bin/python && ln -s $PYTHON /usr/bin/python
+RUN PYTHON3=`ls /usr/share/python3*/dist/python3-arvados-cwl-runner/bin/python3|head -n1` && rm -f /usr/bin/python3 && ln -s $PYTHON3 /usr/bin/python3
 
 # Install dependencies and set up system.
 RUN /usr/sbin/adduser --disabled-password \
diff --git a/go.mod b/go.mod
index 73922de91a..525bae11ee 100644
--- a/go.mod
+++ b/go.mod
@@ -61,7 +61,7 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
-	github.com/containerd/containerd v1.5.8 // indirect
+	github.com/containerd/containerd v1.5.10 // indirect
 	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
 	github.com/docker/go-connections v0.3.0 // indirect
@@ -81,7 +81,7 @@ require (
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/image-spec v1.0.2 // indirect
 	github.com/pelletier/go-buffruneio v0.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
diff --git a/go.sum b/go.sum
index fcccf4aa41..82a8d83d7e 100644
--- a/go.sum
+++ b/go.sum
@@ -180,8 +180,8 @@ github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7
 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
 github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
 github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw=
-github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
+github.com/containerd/containerd v1.5.10 h1:3cQ2uRVCkJVcx5VombsE7105Gl9Wrl7ORAO3+4+ogf4=
+github.com/containerd/containerd v1.5.10/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -548,8 +548,9 @@ github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
-github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=
+github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index 656385cc1c..8bbc33ba08 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -915,11 +915,6 @@ Clusters:
       # If false, containers are scheduled on preemptible instances
       # only when requested by the submitter.
       #
-      # Note that arvados-cwl-runner does not currently offer a
-      # feature to request preemptible instances, so this value
-      # effectively acts as a cluster-wide decision about whether to
-      # use preemptible instances.
-      #
       # This flag is ignored if no preemptible instance types are
       # configured, and has no effect on top-level containers.
       AlwaysUsePreemptibleInstances: true
diff --git a/lib/controller/localdb/collection.go b/lib/controller/localdb/collection.go
index 96c89252ec..868e466e9e 100644
--- a/lib/controller/localdb/collection.go
+++ b/lib/controller/localdb/collection.go
@@ -6,10 +6,17 @@ package localdb
 
 import (
 	"context"
+	"fmt"
+	"net/http"
+	"os"
+	"sort"
+	"strings"
 	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/httpserver"
 )
 
 // CollectionGet defers to railsProxy for everything except blob
@@ -61,6 +68,9 @@ func (conn *Conn) CollectionCreate(ctx context.Context, opts arvados.CreateOptio
 		// them.
 		opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
 	}
+	if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, "", opts.Attrs, opts.ReplaceFiles); err != nil {
+		return arvados.Collection{}, err
+	}
 	resp, err := conn.railsProxy.CollectionCreate(ctx, opts)
 	if err != nil {
 		return resp, err
@@ -82,6 +92,9 @@ func (conn *Conn) CollectionUpdate(ctx context.Context, opts arvados.UpdateOptio
 		// them.
 		opts.Select = append([]string{"is_trashed", "trash_at"}, opts.Select...)
 	}
+	if opts.Attrs, err = conn.applyReplaceFilesOption(ctx, opts.UUID, opts.Attrs, opts.ReplaceFiles); err != nil {
+		return arvados.Collection{}, err
+	}
 	resp, err := conn.railsProxy.CollectionUpdate(ctx, opts)
 	if err != nil {
 		return resp, err
@@ -108,3 +121,147 @@ func (conn *Conn) signCollection(ctx context.Context, coll *arvados.Collection)
 	}
 	coll.ManifestText = arvados.SignManifest(coll.ManifestText, token, exp, ttl, []byte(conn.cluster.Collections.BlobSigningKey))
 }
+
+// If replaceFiles is non-empty, populate attrs["manifest_text"] by
+// starting with the content of fromUUID (or an empty collection if
+// fromUUID is empty) and applying the specified file/directory
+// replacements.
+//
+// Return value is the (possibly modified) attrs map.
+func (conn *Conn) applyReplaceFilesOption(ctx context.Context, fromUUID string, attrs map[string]interface{}, replaceFiles map[string]string) (map[string]interface{}, error) {
+	if len(replaceFiles) == 0 {
+		return attrs, nil
+	} else if mtxt, ok := attrs["manifest_text"].(string); ok && len(mtxt) > 0 {
+		return nil, httpserver.Errorf(http.StatusBadRequest, "ambiguous request: both 'replace_files' and attrs['manifest_text'] values provided")
+	}
+
+	// Load the current collection (if any) and set up an
+	// in-memory filesystem.
+	var dst arvados.Collection
+	if _, replacingRoot := replaceFiles["/"]; !replacingRoot && fromUUID != "" {
+		src, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: fromUUID})
+		if err != nil {
+			return nil, err
+		}
+		dst = src
+	}
+	dstfs, err := dst.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
+	if err != nil {
+		return nil, err
+	}
+
+	// Sort replacements by source collection to avoid redundant
+	// reloads when a source collection is used more than
+	// once. Note empty sources (which mean "delete target path")
+	// sort first.
+	dstTodo := make([]string, 0, len(replaceFiles))
+	{
+		srcid := make(map[string]string, len(replaceFiles))
+		for dst, src := range replaceFiles {
+			dstTodo = append(dstTodo, dst)
+			if i := strings.IndexRune(src, '/'); i > 0 {
+				srcid[dst] = src[:i]
+			}
+		}
+		sort.Slice(dstTodo, func(i, j int) bool {
+			return srcid[dstTodo[i]] < srcid[dstTodo[j]]
+		})
+	}
+
+	// Reject attempt to replace a node as well as its descendant
+	// (e.g., a/ and a/b/), which is unsupported, except where the
+	// source for a/ is empty (i.e., delete).
+	for _, dst := range dstTodo {
+		if dst != "/" && (strings.HasSuffix(dst, "/") ||
+			strings.HasSuffix(dst, "/.") ||
+			strings.HasSuffix(dst, "/..") ||
+			strings.Contains(dst, "//") ||
+			strings.Contains(dst, "/./") ||
+			strings.Contains(dst, "/../") ||
+			!strings.HasPrefix(dst, "/")) {
+			return nil, httpserver.Errorf(http.StatusBadRequest, "invalid replace_files target: %q", dst)
+		}
+		for i := 0; i < len(dst)-1; i++ {
+			if dst[i] != '/' {
+				continue
+			}
+			outerdst := dst[:i]
+			if outerdst == "" {
+				outerdst = "/"
+			}
+			if outersrc := replaceFiles[outerdst]; outersrc != "" {
+				return nil, httpserver.Errorf(http.StatusBadRequest, "replace_files: cannot operate on target %q inside non-empty target %q", dst, outerdst)
+			}
+		}
+	}
+
+	var srcidloaded string
+	var srcfs arvados.FileSystem
+	// Apply the requested replacements.
+	for _, dst := range dstTodo {
+		src := replaceFiles[dst]
+		if src == "" {
+			if dst == "/" {
+				// In this case we started with a
+				// blank manifest, so there can't be
+				// anything to delete.
+				continue
+			}
+			err := dstfs.RemoveAll(dst)
+			if err != nil {
+				return nil, fmt.Errorf("RemoveAll(%s): %w", dst, err)
+			}
+			continue
+		}
+		srcspec := strings.SplitN(src, "/", 2)
+		srcid, srcpath := srcspec[0], "/"
+		if !arvadosclient.PDHMatch(srcid) {
+			return nil, httpserver.Errorf(http.StatusBadRequest, "invalid source %q for replace_files[%q]: must be \"\" or \"PDH\" or \"PDH/path\"", src, dst)
+		}
+		if len(srcspec) == 2 && srcspec[1] != "" {
+			srcpath = srcspec[1]
+		}
+		if srcidloaded != srcid {
+			srcfs = nil
+			srccoll, err := conn.CollectionGet(ctx, arvados.GetOptions{UUID: srcid})
+			if err != nil {
+				return nil, err
+			}
+			// We use StubClient here because we don't
+			// want srcfs to read/write any file data or
+			// sync collection state to/from the database.
+			srcfs, err = srccoll.FileSystem(&arvados.StubClient{}, &arvados.StubClient{})
+			if err != nil {
+				return nil, err
+			}
+			srcidloaded = srcid
+		}
+		snap, err := arvados.Snapshot(srcfs, srcpath)
+		if err != nil {
+			return nil, httpserver.Errorf(http.StatusBadRequest, "error getting snapshot of %q from %q: %w", srcpath, srcid, err)
+		}
+		// Create intermediate dirs, in case dst is
+		// "newdir1/newdir2/dst".
+		for i := 1; i < len(dst)-1; i++ {
+			if dst[i] == '/' {
+				err = dstfs.Mkdir(dst[:i], 0777)
+				if err != nil && !os.IsExist(err) {
+					return nil, httpserver.Errorf(http.StatusBadRequest, "error creating parent dirs for %q: %w", dst, err)
+				}
+			}
+		}
+		err = arvados.Splice(dstfs, dst, snap)
+		if err != nil {
+			return nil, fmt.Errorf("error splicing snapshot onto path %q: %w", dst, err)
+		}
+	}
+	mtxt, err := dstfs.MarshalManifest(".")
+	if err != nil {
+		return nil, err
+	}
+	if attrs == nil {
+		attrs = make(map[string]interface{}, 1)
+	}
+	attrs["manifest_text"] = mtxt
+	return attrs, nil
+}
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index bbfb811165..dac8b769fe 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -6,16 +6,22 @@ package localdb
 
 import (
 	"context"
+	"io/fs"
+	"path/filepath"
 	"regexp"
+	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/controller/rpc"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 	"git.arvados.org/arvados.git/sdk/go/arvadostest"
 	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/keepclient"
 	check "gopkg.in/check.v1"
 )
 
@@ -71,7 +77,7 @@ func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
 	s.localdb.vocabularyCache = voc
 }
 
-func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
+func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
 	s.setUpVocabulary(c, "")
 	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 
@@ -88,6 +94,7 @@ func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
 	for _, tt := range tests {
 		c.Log(c.TestName()+" ", tt.name)
 
+		// Create with properties
 		coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
 			Select: []string{"uuid", "properties"},
 			Attrs: map[string]interface{}{
@@ -99,26 +106,9 @@ func (s *CollectionSuite) TestCollectionCreateWithProperties(c *check.C) {
 		} else {
 			c.Assert(err, check.NotNil)
 		}
-	}
-}
-
-func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
-	s.setUpVocabulary(c, "")
-	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 
-	tests := []struct {
-		name    string
-		props   map[string]interface{}
-		success bool
-	}{
-		{"Invalid prop key", map[string]interface{}{"Priority": "IDVALIMPORTANCES1"}, false},
-		{"Invalid prop value", map[string]interface{}{"IDTAGIMPORTANCES": "high"}, false},
-		{"Valid prop key & value", map[string]interface{}{"IDTAGIMPORTANCES": "IDVALIMPORTANCES1"}, true},
-		{"Empty properties", map[string]interface{}{}, true},
-	}
-	for _, tt := range tests {
-		c.Log(c.TestName()+" ", tt.name)
-		coll, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
+		// Create, then update with properties
+		coll, err = s.localdb.CollectionCreate(ctx, arvados.CreateOptions{})
 		c.Assert(err, check.IsNil)
 		coll, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
 			UUID:   coll.UUID,
@@ -135,6 +125,180 @@ func (s *CollectionSuite) TestCollectionUpdateWithProperties(c *check.C) {
 	}
 }
 
+func (s *CollectionSuite) TestCollectionReplaceFiles(c *check.C) {
+	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.AdminToken}})
+	foo, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"owner_uuid":    arvadostest.ActiveUserUUID,
+			"manifest_text": ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n",
+		}})
+	c.Assert(err, check.IsNil)
+	s.localdb.signCollection(ctx, &foo)
+	foobarbaz, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"owner_uuid":    arvadostest.ActiveUserUUID,
+			"manifest_text": "./foo/bar 73feffa4b7f6bb68e44cf984c85f6e88+3 0:3:baz.txt\n",
+		}})
+	c.Assert(err, check.IsNil)
+	s.localdb.signCollection(ctx, &foobarbaz)
+	wazqux, err := s.localdb.railsProxy.CollectionCreate(ctx, arvados.CreateOptions{
+		Attrs: map[string]interface{}{
+			"owner_uuid":    arvadostest.ActiveUserUUID,
+			"manifest_text": "./waz d85b1213473c2fd7c2045020a6b9c62b+3 0:3:qux.txt\n",
+		}})
+	c.Assert(err, check.IsNil)
+	s.localdb.signCollection(ctx, &wazqux)
+
+	ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+
+	// Create using content from existing collections
+	dst, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+		ReplaceFiles: map[string]string{
+			"/f": foo.PortableDataHash + "/foo.txt",
+			"/b": foobarbaz.PortableDataHash + "/foo/bar",
+			"/q": wazqux.PortableDataHash + "/",
+			"/w": wazqux.PortableDataHash + "/waz",
+		},
+		Attrs: map[string]interface{}{
+			"owner_uuid": arvadostest.ActiveUserUUID,
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "f", "b/baz.txt", "q/waz/qux.txt", "w/qux.txt")
+
+	// Delete a file and a directory
+	dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID: dst.UUID,
+		ReplaceFiles: map[string]string{
+			"/f":     "",
+			"/q/waz": "",
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "b/baz.txt", "q/", "w/qux.txt")
+
+	// Move and copy content within collection
+	dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID: dst.UUID,
+		ReplaceFiles: map[string]string{
+			// Note splicing content to /b/corge.txt but
+			// removing everything else from /b
+			"/b":              "",
+			"/b/corge.txt":    dst.PortableDataHash + "/b/baz.txt",
+			"/quux/corge.txt": dst.PortableDataHash + "/b/baz.txt",
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "b/corge.txt", "q/", "w/qux.txt", "quux/corge.txt")
+
+	// Remove everything except one file
+	dst, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID: dst.UUID,
+		ReplaceFiles: map[string]string{
+			"/":            "",
+			"/b/corge.txt": dst.PortableDataHash + "/b/corge.txt",
+		}})
+	c.Assert(err, check.IsNil)
+	s.expectFiles(c, dst, "b/corge.txt")
+
+	// Copy entire collection to root
+	dstcopy, err := s.localdb.CollectionCreate(ctx, arvados.CreateOptions{
+		ReplaceFiles: map[string]string{
+			"/": dst.PortableDataHash,
+		}})
+	c.Check(err, check.IsNil)
+	c.Check(dstcopy.PortableDataHash, check.Equals, dst.PortableDataHash)
+	s.expectFiles(c, dstcopy, "b/corge.txt")
+
+	// Check invalid targets, sources, and combinations
+	for _, badrepl := range []map[string]string{
+		{
+			"/foo/nope": dst.PortableDataHash + "/b",
+			"/foo":      dst.PortableDataHash + "/b",
+		},
+		{
+			"/foo":      dst.PortableDataHash + "/b",
+			"/foo/nope": "",
+		},
+		{
+			"/":     dst.PortableDataHash + "/",
+			"/nope": "",
+		},
+		{
+			"/":     dst.PortableDataHash + "/",
+			"/nope": dst.PortableDataHash + "/b",
+		},
+		{"/bad/": ""},
+		{"/./bad": ""},
+		{"/b/./ad": ""},
+		{"/b/../ad": ""},
+		{"/b/.": ""},
+		{".": ""},
+		{"bad": ""},
+		{"": ""},
+		{"/bad": "/b"},
+		{"/bad": "bad/b"},
+		{"/bad": dst.UUID + "/b"},
+	} {
+		_, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+			UUID:         dst.UUID,
+			ReplaceFiles: badrepl,
+		})
+		c.Logf("badrepl %#v\n... got err: %s", badrepl, err)
+		c.Check(err, check.NotNil)
+	}
+
+	// Check conflicting replace_files and manifest_text
+	_, err = s.localdb.CollectionUpdate(ctx, arvados.UpdateOptions{
+		UUID:         dst.UUID,
+		ReplaceFiles: map[string]string{"/": ""},
+		Attrs: map[string]interface{}{
+			"manifest_text": ". d41d8cd98f00b204e9800998ecf8427e+0 0:0:z\n",
+		}})
+	c.Logf("replace_files+manifest_text\n... got err: %s", err)
+	c.Check(err, check.ErrorMatches, "ambiguous request: both.*replace_files.*manifest_text.*")
+}
+
+// expectFiles checks coll's directory structure against the given
+// list of expected files and empty directories. An expected path with
+// a trailing slash indicates an empty directory.
+func (s *CollectionSuite) expectFiles(c *check.C, coll arvados.Collection, expected ...string) {
+	client := arvados.NewClientFromEnv()
+	ac, err := arvadosclient.New(client)
+	c.Assert(err, check.IsNil)
+	kc, err := keepclient.MakeKeepClient(ac)
+	c.Assert(err, check.IsNil)
+	cfs, err := coll.FileSystem(arvados.NewClientFromEnv(), kc)
+	c.Assert(err, check.IsNil)
+	var found []string
+	nonemptydirs := map[string]bool{}
+	fs.WalkDir(arvados.FS(cfs), "/", func(path string, d fs.DirEntry, err error) error {
+		dir, _ := filepath.Split(path)
+		nonemptydirs[dir] = true
+		if d.IsDir() {
+			if path != "/" {
+				path += "/"
+			}
+			if !nonemptydirs[path] {
+				nonemptydirs[path] = false
+			}
+		} else {
+			found = append(found, path)
+		}
+		return nil
+	})
+	for d, nonempty := range nonemptydirs {
+		if !nonempty {
+			found = append(found, d)
+		}
+	}
+	for i, path := range found {
+		if path != "/" {
+			found[i] = strings.TrimPrefix(path, "/")
+		}
+	}
+	sort.Strings(found)
+	sort.Strings(expected)
+	c.Check(found, check.DeepEquals, expected)
+}
+
 func (s *CollectionSuite) TestSignatures(c *check.C) {
 	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
 
diff --git a/lib/controller/localdb/login_oidc.go b/lib/controller/localdb/login_oidc.go
index e076f7e128..6d6f80f39c 100644
--- a/lib/controller/localdb/login_oidc.go
+++ b/lib/controller/localdb/login_oidc.go
@@ -31,6 +31,7 @@ import (
 	"github.com/coreos/go-oidc"
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/oauth2"
 	"google.golang.org/api/option"
@@ -43,6 +44,7 @@ var (
 	tokenCacheNegativeTTL = time.Minute * 5
 	tokenCacheTTL         = time.Minute * 10
 	tokenCacheRaceWindow  = time.Minute
+	pqCodeUniqueViolation = pq.ErrorCode("23505")
 )
 
 type oidcLoginController struct {
@@ -479,7 +481,6 @@ func (ta *oidcTokenAuthorizer) registerToken(ctx context.Context, tok string) er
 	// it's expiring.
 	exp := time.Now().UTC().Add(tokenCacheTTL + tokenCacheRaceWindow)
 
-	var aca arvados.APIClientAuthorization
 	if updating {
 		_, err = tx.ExecContext(ctx, `update api_client_authorizations set expires_at=$1 where api_token=$2`, exp, hmac)
 		if err != nil {
@@ -487,23 +488,44 @@ func (ta *oidcTokenAuthorizer) registerToken(ctx context.Context, tok string) er
 		}
 		ctxlog.FromContext(ctx).WithField("HMAC", hmac).Debug("(*oidcTokenAuthorizer)registerToken: updated api_client_authorizations row")
 	} else {
-		aca, err = ta.ctrl.Parent.CreateAPIClientAuthorization(ctx, ta.ctrl.Cluster.SystemRootToken, *authinfo)
+		aca, err := ta.ctrl.Parent.CreateAPIClientAuthorization(ctx, ta.ctrl.Cluster.SystemRootToken, *authinfo)
 		if err != nil {
 			return err
 		}
-		_, err = tx.ExecContext(ctx, `update api_client_authorizations set api_token=$1, expires_at=$2 where uuid=$3`, hmac, exp, aca.UUID)
+		_, err = tx.ExecContext(ctx, `savepoint upd`)
 		if err != nil {
+			return err
+		}
+		_, err = tx.ExecContext(ctx, `update api_client_authorizations set api_token=$1, expires_at=$2 where uuid=$3`, hmac, exp, aca.UUID)
+		if e, ok := err.(*pq.Error); ok && e.Code == pqCodeUniqueViolation {
+			// unique_violation, given that the above
+			// query did not find a row with matching
+			// api_token, means another thread/process
+			// also received this same token and won the
+			// race to insert it -- in which case this
+			// thread doesn't need to update the database.
+			// Discard the redundant row.
+			_, err = tx.ExecContext(ctx, `rollback to savepoint upd`)
+			if err != nil {
+				return err
+			}
+			_, err = tx.ExecContext(ctx, `delete from api_client_authorizations where uuid=$1`, aca.UUID)
+			if err != nil {
+				return err
+			}
+			ctxlog.FromContext(ctx).WithField("HMAC", hmac).Debug("(*oidcTokenAuthorizer)registerToken: api_client_authorizations row inserted by another thread")
+		} else if err != nil {
+			ctxlog.FromContext(ctx).Errorf("%#v", err)
 			return fmt.Errorf("error adding OIDC access token to database: %w", err)
+		} else {
+			ctxlog.FromContext(ctx).WithFields(logrus.Fields{"UUID": aca.UUID, "HMAC": hmac}).Debug("(*oidcTokenAuthorizer)registerToken: inserted api_client_authorizations row")
 		}
-		aca.APIToken = hmac
-		ctxlog.FromContext(ctx).WithFields(logrus.Fields{"UUID": aca.UUID, "HMAC": hmac}).Debug("(*oidcTokenAuthorizer)registerToken: inserted api_client_authorizations row")
 	}
 	err = tx.Commit()
 	if err != nil {
 		return err
 	}
-	aca.ExpiresAt = exp
-	ta.cache.Add(tok, aca)
+	ta.cache.Add(tok, arvados.APIClientAuthorization{ExpiresAt: exp})
 	return nil
 }
 
diff --git a/lib/controller/localdb/login_oidc_test.go b/lib/controller/localdb/login_oidc_test.go
index 4778e45f5f..b9f0f56e05 100644
--- a/lib/controller/localdb/login_oidc_test.go
+++ b/lib/controller/localdb/login_oidc_test.go
@@ -17,6 +17,7 @@ import (
 	"net/url"
 	"sort"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -236,18 +237,49 @@ func (s *OIDCLoginSuite) TestOIDCAuthorizer(c *check.C) {
 
 	ctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{accessToken}})
 	var exp1 time.Time
-	oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
-		creds, ok := auth.FromContext(ctx)
-		c.Assert(ok, check.Equals, true)
-		c.Assert(creds.Tokens, check.HasLen, 1)
-		c.Check(creds.Tokens[0], check.Equals, accessToken)
 
-		err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp1)
-		c.Check(err, check.IsNil)
-		c.Check(exp1.Sub(time.Now()) > -time.Second, check.Equals, true)
-		c.Check(exp1.Sub(time.Now()) < time.Second, check.Equals, true)
-		return nil, nil
-	})(ctx, nil)
+	concurrent := 4
+	s.fakeProvider.HoldUserInfo = make(chan *http.Request)
+	s.fakeProvider.ReleaseUserInfo = make(chan struct{})
+	go func() {
+		for i := 0; ; i++ {
+			if i == concurrent {
+				close(s.fakeProvider.ReleaseUserInfo)
+			}
+			<-s.fakeProvider.HoldUserInfo
+		}
+	}()
+	var wg sync.WaitGroup
+	for i := 0; i < concurrent; i++ {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			_, err := oidcAuthorizer.WrapCalls(func(ctx context.Context, opts interface{}) (interface{}, error) {
+				c.Logf("concurrent req %d/%d", i, concurrent)
+				var exp time.Time
+
+				creds, ok := auth.FromContext(ctx)
+				c.Assert(ok, check.Equals, true)
+				c.Assert(creds.Tokens, check.HasLen, 1)
+				c.Check(creds.Tokens[0], check.Equals, accessToken)
+
+				err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp)
+				c.Check(err, check.IsNil)
+				c.Check(exp.Sub(time.Now()) > -time.Second, check.Equals, true)
+				c.Check(exp.Sub(time.Now()) < time.Second, check.Equals, true)
+				if i == 0 {
+					exp1 = exp
+				}
+				return nil, nil
+			})(ctx, nil)
+			c.Check(err, check.IsNil)
+		}()
+	}
+	wg.Wait()
+	if c.Failed() {
+		c.Fatal("giving up")
+	}
 
 	// If the token is used again after the in-memory cache
 	// expires, oidcAuthorizer must re-check the token and update
@@ -257,8 +289,8 @@ func (s *OIDCLoginSuite) TestOIDCAuthorizer(c *check.C) {
 		var exp time.Time
 		err := db.QueryRowContext(ctx, `select expires_at at time zone 'UTC' from api_client_authorizations where api_token=$1`, apiToken).Scan(&exp)
 		c.Check(err, check.IsNil)
-		c.Check(exp.Sub(exp1) > 0, check.Equals, true)
-		c.Check(exp.Sub(exp1) < time.Second, check.Equals, true)
+		c.Check(exp.Sub(exp1) > 0, check.Equals, true, check.Commentf("expect %v > 0", exp.Sub(exp1)))
+		c.Check(exp.Sub(exp1) < time.Second, check.Equals, true, check.Commentf("expect %v < 1s", exp.Sub(exp1)))
 		return nil, nil
 	})(ctx, nil)
 
diff --git a/lib/controller/localdb/login_pam.go b/lib/controller/localdb/login_pam.go
index 237f900a83..14e0a582c1 100644
--- a/lib/controller/localdb/login_pam.go
+++ b/lib/controller/localdb/login_pam.go
@@ -2,6 +2,8 @@
 //
 // SPDX-License-Identifier: AGPL-3.0
 
+//go:build !static
+
 package localdb
 
 import (
diff --git a/lib/controller/localdb/login_pam_static.go b/lib/controller/localdb/login_pam_static.go
new file mode 100644
index 0000000000..420a256da1
--- /dev/null
+++ b/lib/controller/localdb/login_pam_static.go
@@ -0,0 +1,31 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+//go:build static
+
+package localdb
+
+import (
+	"context"
+	"errors"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+type pamLoginController struct {
+	Cluster *arvados.Cluster
+	Parent  *Conn
+}
+
+func (ctrl *pamLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+	return logout(ctx, ctrl.Cluster, opts)
+}
+
+func (ctrl *pamLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+	return arvados.LoginResponse{}, errors.New("interactive login is not available")
+}
+
+func (ctrl *pamLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+	return arvados.APIClientAuthorization{}, errors.New("support not available due to static compilation")
+}
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 4fa3f26ab5..65f43e9644 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -19,6 +19,7 @@ import (
 	"os"
 	"os/exec"
 	"os/signal"
+	"os/user"
 	"path"
 	"path/filepath"
 	"regexp"
@@ -1475,6 +1476,7 @@ func (runner *ContainerRunner) NewArvLogWriter(name string) (io.WriteCloser, err
 // Run the full container lifecycle.
 func (runner *ContainerRunner) Run() (err error) {
 	runner.CrunchLog.Printf("crunch-run %s started", cmd.Version.String())
+	runner.CrunchLog.Printf("%s", currentUserAndGroups())
 	runner.CrunchLog.Printf("Executing container '%s' using %s runtime", runner.Container.UUID, runner.executor.Runtime())
 
 	hostname, hosterr := os.Hostname()
@@ -2045,3 +2047,30 @@ func startLocalKeepstore(configData ConfigData, logbuf io.Writer) (*exec.Cmd, er
 	os.Setenv("ARVADOS_KEEP_SERVICES", url)
 	return cmd, nil
 }
+
+// return current uid, gid, groups in a format suitable for logging:
+// "crunch-run process has uid=1234(arvados) gid=1234(arvados)
+// groups=1234(arvados),114(fuse)"
+func currentUserAndGroups() string {
+	u, err := user.Current()
+	if err != nil {
+		return fmt.Sprintf("error getting current user ID: %s", err)
+	}
+	s := fmt.Sprintf("crunch-run process has uid=%s(%s) gid=%s", u.Uid, u.Username, u.Gid)
+	if g, err := user.LookupGroupId(u.Gid); err == nil {
+		s += fmt.Sprintf("(%s)", g.Name)
+	}
+	s += " groups="
+	if gids, err := u.GroupIds(); err == nil {
+		for i, gid := range gids {
+			if i > 0 {
+				s += ","
+			}
+			s += gid
+			if g, err := user.LookupGroupId(gid); err == nil {
+				s += fmt.Sprintf("(%s)", g.Name)
+			}
+		}
+	}
+	return s
+}
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 26f78d2bf7..62df0032b4 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -885,6 +885,7 @@ func (s *TestSuite) TestLogVersionAndRuntime(c *C) {
 
 	c.Assert(s.api.Logs["crunch-run"], NotNil)
 	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run \S+ \(go\S+\) start.*`)
+	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*crunch-run process has uid=\d+\(.+\) gid=\d+\(.+\) groups=\d+\(.+\)(,\d+\(.+\))*\n.*`)
 	c.Check(s.api.Logs["crunch-run"].String(), Matches, `(?ms).*Executing container 'zzzzz-zzzzz-zzzzzzzzzzzzzzz' using stub runtime.*`)
 }
 
diff --git a/lib/crunchstat/crunchstat.go b/lib/crunchstat/crunchstat.go
index 028083fa0d..10cd7cfce4 100644
--- a/lib/crunchstat/crunchstat.go
+++ b/lib/crunchstat/crunchstat.go
@@ -21,16 +21,6 @@ import (
 	"time"
 )
 
-// This magically allows us to look up userHz via _SC_CLK_TCK:
-
-/*
-#include 
-#include 
-#include 
-#include 
-*/
-import "C"
-
 // A Reporter gathers statistics for a cgroup and writes them to a
 // log.Logger.
 type Reporter struct {
@@ -395,7 +385,7 @@ func (r *Reporter) doCPUStats() {
 
 	var userTicks, sysTicks int64
 	fmt.Sscanf(string(b), "user %d\nsystem %d", &userTicks, &sysTicks)
-	userHz := float64(C.sysconf(C._SC_CLK_TCK))
+	userHz := float64(100)
 	nextSample := cpuSample{
 		hasData:    true,
 		sampleTime: time.Now(),
diff --git a/sdk/cli/test/test_arv-collection-create.rb b/sdk/cli/test/test_arv-collection-create.rb
index 39c50bcc83..1b5a368b7d 100644
--- a/sdk/cli/test/test_arv-collection-create.rb
+++ b/sdk/cli/test/test_arv-collection-create.rb
@@ -14,14 +14,48 @@ class TestCollectionCreate < Minitest::Test
 
   def test_small_collection
     uuid = Digest::MD5.hexdigest(foo_manifest) + '+' + foo_manifest.size.to_s
+    ok = nil
     out, err = capture_subprocess_io do
-      assert_arv('--format', 'uuid', 'collection', 'create', '--collection', {
-                   uuid: uuid,
-                   manifest_text: foo_manifest
-                 }.to_json)
+      ok = arv('--format', 'uuid', 'collection', 'create', '--collection', {
+                     uuid: uuid,
+                     manifest_text: foo_manifest
+                   }.to_json)
     end
-    assert(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/.match(out))
-    assert_equal '', err
+    assert_equal('', err)
+    assert_equal(true, ok)
+    assert_match(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/, out)
+  end
+
+  def test_collection_replace_files
+    ok = nil
+    uuid, err = capture_subprocess_io do
+      ok = arv('--format', 'uuid', 'collection', 'create', '--collection', '{}')
+    end
+    assert_equal('', err)
+    assert_equal(true, ok)
+    assert_match(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/, uuid)
+    uuid = uuid.strip
+
+    out, err = capture_subprocess_io do
+      ok = arv('--format', 'uuid',
+                   'collection', 'update',
+                   '--uuid', uuid,
+                   '--collection', '{}',
+                   '--replace-files', {
+                     "/gpl.pdf": "b519d9cb706a29fc7ea24dbea2f05851+93/GNU_General_Public_License,_version_3.pdf",
+                   }.to_json)
+    end
+    assert_equal('', err)
+    assert_equal(true, ok)
+    assert_equal(uuid, out.strip)
+
+    ok = nil
+    out, err = capture_subprocess_io do
+      ok = arv('--format', 'json', 'collection', 'get', '--uuid', uuid)
+    end
+    assert_equal('', err)
+    assert_equal(true, ok)
+    assert_match(/\. 6a4ff0499484c6c79c95cd8c566bd25f\+249025.* 0:249025:gpl.pdf\\n/, out)
   end
 
   def test_read_resource_object_from_file
@@ -29,29 +63,22 @@ class TestCollectionCreate < Minitest::Test
     begin
       tempfile.write({manifest_text: foo_manifest}.to_json)
       tempfile.close
+      ok = nil
       out, err = capture_subprocess_io do
-        assert_arv('--format', 'uuid',
-                   'collection', 'create', '--collection', tempfile.path)
+        ok = arv('--format', 'uuid',
+                     'collection', 'create', '--collection', tempfile.path)
       end
-      assert(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/.match(out))
-      assert_equal '', err
+      assert_equal('', err)
+      assert_equal(true, ok)
+      assert_match(/^([0-9a-z]{5}-4zz18-[0-9a-z]{15})?$/, out)
     ensure
       tempfile.unlink
     end
   end
 
   protected
-  def assert_arv(*args)
-    expect = case args.first
-             when true, false
-               args.shift
-             else
-               true
-             end
-    assert_equal(expect,
-                 system(['./bin/arv', 'arv'], *args),
-                 "`arv #{args.join ' '}` " +
-                 "should exit #{if expect then 0 else 'non-zero' end}")
+  def arv(*args)
+    system(['./bin/arv', 'arv'], *args)
   end
 
   def foo_manifest
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 826467cc09..c73b358ecc 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -213,6 +213,10 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--http-timeout", type=int,
                         default=5*60, dest="http_timeout", help="API request timeout in seconds. Default is 300 seconds (5 minutes).")
 
+    exgroup = parser.add_mutually_exclusive_group()
+    exgroup.add_argument("--enable-preemptible", dest="enable_preemptible", default=None, action="store_true", help="Use preemptible instances. Control individual steps with arv:UsePreemptible hint.")
+    exgroup.add_argument("--disable-preemptible", dest="enable_preemptible", default=None, action="store_false", help="Don't use preemptible instances.")
+
     parser.add_argument(
         "--skip-schemas",
         action="store_true",
@@ -255,7 +259,8 @@ def add_arv_hints():
         "http://arvados.org/cwl#ClusterTarget",
         "http://arvados.org/cwl#OutputStorageClass",
         "http://arvados.org/cwl#ProcessProperties",
-        "http://commonwl.org/cwltool#CUDARequirement"
+        "http://commonwl.org/cwltool#CUDARequirement",
+        "http://arvados.org/cwl#UsePreemptible",
     ])
 
 def exit_signal_handler(sigcode, frame):
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
index d5efa31a00..af75481431 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.0.yml
@@ -359,13 +359,44 @@ $graph:
 
         See https://docs.nvidia.com/deploy/cuda-compatibility/ for
         details.
-    cudaComputeCapabilityMin:
-      type: string
-      doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
-    deviceCountMin:
-      type: int?
+    cudaComputeCapability:
+      type:
+        - 'string'
+        - 'string[]'
+      doc: |
+        CUDA hardware capability required to run the software, in X.Y
+        format.
+
+        * If this is a single value, it defines only the minimum
+          compute capability.  GPUs with higher capability are also
+          accepted.
+
+        * If it is an array value, then only select GPUs with compute
+          capabilities that explicitly appear in the array.
+    cudaDeviceCountMin:
+      type: ['null', int, cwl:Expression]
       default: 1
-      doc: Minimum number of GPU devices to request, default 1.
-    deviceCountMax:
-      type: int?
-      doc: Maximum number of GPU devices to request.  If not specified, same as `deviceCountMin`.
+      doc: |
+        Minimum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMax`.  If neither are specified,
+        default 1.
+    cudaDeviceCountMax:
+      type: ['null', int, cwl:Expression]
+      doc: |
+        Maximum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMin`.
+
+- name: UsePreemptible
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Specify a workflow step should opt-in or opt-out of using preemptible (spot) instances.
+  fields:
+    class:
+      type: string
+      doc: "Always 'arv:UsePreemptible"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    usePreemptible: boolean
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
index 4a6b6947ff..0ae451ccaa 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.1.yml
@@ -302,13 +302,44 @@ $graph:
 
         See https://docs.nvidia.com/deploy/cuda-compatibility/ for
         details.
-    cudaComputeCapabilityMin:
-      type: string
-      doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
-    deviceCountMin:
-      type: int?
+    cudaComputeCapability:
+      type:
+        - 'string'
+        - 'string[]'
+      doc: |
+        CUDA hardware capability required to run the software, in X.Y
+        format.
+
+        * If this is a single value, it defines only the minimum
+          compute capability.  GPUs with higher capability are also
+          accepted.
+
+        * If it is an array value, then only select GPUs with compute
+          capabilities that explicitly appear in the array.
+    cudaDeviceCountMin:
+      type: ['null', int, cwl:Expression]
       default: 1
-      doc: Minimum number of GPU devices to request, default 1.
-    deviceCountMax:
-      type: int?
-      doc: Maximum number of GPU devices to request.  If not specified, same as `deviceCountMin`.
+      doc: |
+        Minimum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMax`.  If neither are specified,
+        default 1.
+    cudaDeviceCountMax:
+      type: ['null', int, cwl:Expression]
+      doc: |
+        Maximum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMin`.
+
+- name: UsePreemptible
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Specify a workflow step should opt-in or opt-out of using preemptible (spot) instances.
+  fields:
+    class:
+      type: string
+      doc: "Always 'arv:UsePreemptible"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    usePreemptible: boolean
diff --git a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
index e95b6543fd..de5e55ca01 100644
--- a/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
+++ b/sdk/cwl/arvados_cwl/arv-cwl-schema-v1.2.yml
@@ -304,13 +304,44 @@ $graph:
 
         See https://docs.nvidia.com/deploy/cuda-compatibility/ for
         details.
-    cudaComputeCapabilityMin:
-      type: string
-      doc: Minimum CUDA hardware capability required to run the software, in X.Y format.
-    deviceCountMin:
-      type: int?
+    cudaComputeCapability:
+      type:
+        - 'string'
+        - 'string[]'
+      doc: |
+        CUDA hardware capability required to run the software, in X.Y
+        format.
+
+        * If this is a single value, it defines only the minimum
+          compute capability.  GPUs with higher capability are also
+          accepted.
+
+        * If it is an array value, then only select GPUs with compute
+          capabilities that explicitly appear in the array.
+    cudaDeviceCountMin:
+      type: ['null', int, cwl:Expression]
       default: 1
-      doc: Minimum number of GPU devices to request, default 1.
-    deviceCountMax:
-      type: int?
-      doc: Maximum number of GPU devices to request.  If not specified, same as `deviceCountMin`.
+      doc: |
+        Minimum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMax`.  If neither are specified,
+        default 1.
+    cudaDeviceCountMax:
+      type: ['null', int, cwl:Expression]
+      doc: |
+        Maximum number of GPU devices to request.  If not specified,
+        same as `cudaDeviceCountMin`.
+
+- name: UsePreemptible
+  type: record
+  extends: cwl:ProcessRequirement
+  inVocab: false
+  doc: |
+    Specify a workflow step should opt-in or opt-out of using preemptible (spot) instances.
+  fields:
+    class:
+      type: string
+      doc: "Always 'arv:UsePreemptible"
+      jsonldPredicate:
+        _id: "@type"
+        _type: "@vocab"
+    usePreemptible: boolean
diff --git a/sdk/cwl/arvados_cwl/arvcontainer.py b/sdk/cwl/arvados_cwl/arvcontainer.py
index 753c2c2502..8c468dd22d 100644
--- a/sdk/cwl/arvados_cwl/arvcontainer.py
+++ b/sdk/cwl/arvados_cwl/arvcontainer.py
@@ -295,11 +295,22 @@ class ArvadosContainer(JobBase):
         cuda_req, _ = self.get_requirement("http://commonwl.org/cwltool#CUDARequirement")
         if cuda_req:
             runtime_constraints["cuda"] = {
-                "device_count": cuda_req.get("deviceCountMin", 1),
+                "device_count": resources.get("cudaDeviceCount", 1),
                 "driver_version": cuda_req["cudaVersionMin"],
-                "hardware_capability": cuda_req["cudaComputeCapabilityMin"]
+                "hardware_capability": aslist(cuda_req["cudaComputeCapability"])[0]
             }
 
+        if runtimeContext.enable_preemptible is False:
+            scheduling_parameters["preemptible"] = False
+        else:
+            preemptible_req, _ = self.get_requirement("http://arvados.org/cwl#UsePreemptible")
+            if preemptible_req:
+                scheduling_parameters["preemptible"] = preemptible_req["usePreemptible"]
+            elif runtimeContext.enable_preemptible is True:
+                scheduling_parameters["preemptible"] = True
+            elif runtimeContext.enable_preemptible is None:
+                pass
+
         if self.timelimit is not None and self.timelimit > 0:
             scheduling_parameters["max_run_time"] = self.timelimit
 
@@ -550,6 +561,12 @@ class RunnerContainer(Runner):
         if self.enable_dev:
             command.append("--enable-dev")
 
+        if runtimeContext.enable_preemptible is True:
+            command.append("--enable-preemptible")
+
+        if runtimeContext.enable_preemptible is False:
+            command.append("--disable-preemptible")
+
         command.extend([workflowpath, "/var/lib/cwl/cwl.input.json"])
 
         container_req["command"] = command
diff --git a/sdk/cwl/arvados_cwl/context.py b/sdk/cwl/arvados_cwl/context.py
index 4239dd3b51..316250106b 100644
--- a/sdk/cwl/arvados_cwl/context.py
+++ b/sdk/cwl/arvados_cwl/context.py
@@ -37,6 +37,7 @@ class ArvRuntimeContext(RuntimeContext):
         self.always_submit_runner = False
         self.collection_cache_size = 256
         self.match_local_docker = False
+        self.enable_preemptible = None
 
         super(ArvRuntimeContext, self).__init__(kwargs)
 
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index ad17950a2f..38e2c4d806 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -40,7 +40,7 @@ import schema_salad.validate as validate
 
 import arvados.collection
 from .util import collectionUUID
-import ruamel.yaml as yaml
+from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 import arvados_cwl.arvdocker
@@ -265,7 +265,8 @@ def upload_dependencies(arvrunner, name, document_loader,
                 textIO = StringIO(text.decode('utf-8'))
             else:
                 textIO = StringIO(text)
-            return yaml.safe_load(textIO)
+            yamlloader = YAML(typ='safe', pure=True)
+            return yamlloader.load(textIO)
         else:
             return {}
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index e126d170b7..c885ebd4b1 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -36,7 +36,7 @@ setup(name='arvados-cwl-runner',
       # file to determine what version of cwltool and schema-salad to
       # build.
       install_requires=[
-          'cwltool==3.1.20220217222804',
+          'cwltool==3.1.20220224085855',
           'schema-salad==8.2.20211116214159',
           'arvados-python-client{}'.format(pysdk_dep),
           'setuptools',
diff --git a/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz b/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz b/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz b/sdk/cwl/tests/chipseq/DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/chip-seq-single.json b/sdk/cwl/tests/chipseq/chip-seq-single.json
new file mode 100644
index 0000000000..758390e8b7
--- /dev/null
+++ b/sdk/cwl/tests/chipseq/chip-seq-single.json
@@ -0,0 +1,99 @@
+{
+    "referenceGenomeSequence": {
+        "class": "File",
+        "location": "data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa",
+        "metadata": {
+            "reference_genome": {
+                "organism": "Homo sapiens",
+                "version": "hg38"
+            },
+            "annotation": {
+                "source": "gencode",
+                "version": "v24"
+            }
+        }
+    },
+    "referenceGenomeSequenceDrosophila": {
+        "class": "File",
+        "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa",
+        "metadata": {
+            "reference_genome": {
+                "organism": "Drosophila melanogaster",
+                "version": "rmel_r6.16"
+            }
+        }
+    },
+    "blacklistBed": {
+        "class": "File",
+        "location": "data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed",
+        "metadata": {
+            "reference_genome": {
+                "organism": "Homo sapiens",
+                "version": "hg38"
+            },
+            "annotation": {
+                "source": "gencode",
+                "version": "v24"
+            }
+        }
+    },
+    "BowtieHumanReference": {
+        "class": "Directory",
+        "location": "data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/",
+        "metadata": {
+            "reference_genome": {
+                "organism": "Homo sapiens",
+                "version": "hg38"
+            },
+            "annotation": {
+                "source": "gencode",
+                "version": "v24"
+            }
+        }
+    },
+    "BowtieDrosophilaReference": {
+        "class": "Directory",
+        "location": "data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/",
+        "metadata": {
+            "reference_genome": {
+                "organism": "Drosophila melanogaster",
+                "version": "rmel_r6.16"
+            }
+        }
+    },
+    "sampleName": "LED054_0p03nMR1.0",
+    "inputFastq1": {
+        "class": "File",
+        "metadata": {
+            "user": "kmavrommatis",
+            "sample_id": [
+               2
+            ]
+        },
+        "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R1.fastq.gz",
+        "secondaryFiles": []
+    },
+    "inputFastq2": {
+        "class": "File",
+        "metadata": {
+            "user": "kmavrommatis",
+            "sample_id": [
+                2
+            ]
+        },
+        "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R3.fastq.gz",
+        "secondaryFiles": []
+    },
+    "inputFastqUMI": {
+        "class": "File",
+        "metadata": {
+            "user": "kmavrommatis",
+            "sample_id": [
+               2
+            ]
+        },
+        "location": "DATEST/ChIP-Seq/Raw/fastq/Input_R2.fastq.gz",
+        "secondaryFiles": []
+    }
+}
+
diff --git a/sdk/cwl/tests/chipseq/cwl-packed.json b/sdk/cwl/tests/chipseq/cwl-packed.json
new file mode 100644
index 0000000000..8921bcbe54
--- /dev/null
+++ b/sdk/cwl/tests/chipseq/cwl-packed.json
@@ -0,0 +1,94 @@
+{
+    "$graph": [
+        {
+            "class": "Workflow",
+            "id": "#main",
+            "doc": "Pipeline that is applied on single ChIP-seq samples.\n\nStarts with QC on the reads and trimming (for adapters and based on quality)\n\nAligns to human genome and adds UMI\n\nAligns to Drosophila genome and counts the number of reads.\n\nAfter the alignment to human genome the files are filtered for duplicates, multimappers and alignments in black listed regions",
+            "label": "ChIP-Seq (single sample)",
+            "inputs": [
+                {
+                    "id": "#inputFastq1",
+                    "type": "File",
+                    "https://www.sevenbridges.com/fileTypes": "fastq",
+                    "https://www.sevenbridges.com/x": 0,
+                    "https://www.sevenbridges.com/y": 1726.25
+                },
+                {
+                    "id": "#blacklistBed",
+                    "type": "File",
+                    "https://www.sevenbridges.com/x": 746.4744873046875,
+                    "https://www.sevenbridges.com/y": 1903.265625
+                },
+                {
+                    "id": "#referenceGenomeSequence",
+                    "type": "File",
+                    "secondaryFiles": [
+                        ".fai",
+                        "^.dict"
+                    ],
+                    "https://www.sevenbridges.com/fileTypes": "fasta, fa",
+                    "https://www.sevenbridges.com/x": 0,
+                    "https://www.sevenbridges.com/y": 1405.203125
+                },
+                {
+                    "id": "#sampleName",
+                    "type": "string",
+                    "https://www.sevenbridges.com/x": 0,
+                    "https://www.sevenbridges.com/y": 1191.171875
+                },
+                {
+                    "id": "#inputFastq2",
+                    "type": [
+                        "null",
+                        "File"
+                    ],
+                    "https://www.sevenbridges.com/fileTypes": "fastq",
+                    "https://www.sevenbridges.com/x": 0,
+                    "https://www.sevenbridges.com/y": 1619.234375
+                },
+                {
+                    "id": "#inputFastqUMI",
+                    "type": "File",
+                    "https://www.sevenbridges.com/x": 0,
+                    "https://www.sevenbridges.com/y": 1512.21875
+                },
+                {
+                    "id": "#BowtieHumanReference",
+                    "type": "Directory",
+                    "https://www.sevenbridges.com/x": 363.875,
+                    "https://www.sevenbridges.com/y": 1519.21875
+                },
+                {
+                    "id": "#BowtieDrosophilaReference",
+                    "type": "Directory",
+                    "https://www.sevenbridges.com/x": 363.875,
+                    "https://www.sevenbridges.com/y": 1626.234375
+                },
+                {
+                    "id": "#referenceGenomeSequenceDrosophila",
+                    "type": "File",
+                    "secondaryFiles": [
+                        ".fai"
+                    ],
+                    "https://www.sevenbridges.com/x": 0,
+                    "https://www.sevenbridges.com/y": 1298.1875
+                }
+            ],
+            "outputs": [
+            ],
+            "steps": [
+                {
+                    "id": "#step1",
+                    "in": {
+                        "inp": "#inputFastq1"
+                    },
+                    "out": [],
+                    "run": "../cat.cwl"
+                }
+            ],
+            "requirements": [
+            ]
+        },
+   ],
+    "cwlVersion": "v1.0"
+}
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed b/sdk/cwl/tests/chipseq/data/Genomes/Blacklist/lists2/hg38-blacklist.v2.bed
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/Bowtie2Index/genome.fa
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Drosophila_melanogaster/dmel_r6.16/WholeGenome/genome.fa
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/Bowtie2Index/genome.fa
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa b/sdk/cwl/tests/chipseq/data/Genomes/Homo_sapiens/GRCh38.p2/WholeGenome/genome.fa
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/sdk/cwl/tests/test_container.py b/sdk/cwl/tests/test_container.py
index 72774daba3..798c5af289 100644
--- a/sdk/cwl/tests/test_container.py
+++ b/sdk/cwl/tests/test_container.py
@@ -16,6 +16,7 @@ import mock
 import unittest
 import os
 import functools
+import threading
 import cwltool.process
 import cwltool.secrets
 import cwltool.load_tool
@@ -75,7 +76,9 @@ class TestContainer(unittest.TestCase):
              "basedir": "",
              "make_fs_access": make_fs_access,
              "construct_tool_object": runner.arv_make_tool,
-             "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access)
+             "fetcher_constructor": functools.partial(arvados_cwl.CollectionFetcher, api_client=runner.api, fs_access=fs_access),
+             "loader": Loader({}),
+             "metadata": cmap({"cwlVersion": INTERNAL_VERSION, "http://commonwl.org/cwltool#original_cwlVersion": "v1.0"})
              })
         runtimeContext = arvados_cwl.context.ArvRuntimeContext(
             {"work_api": "containers",
@@ -83,9 +86,11 @@ class TestContainer(unittest.TestCase):
              "name": "test_run_"+str(enable_reuse),
              "make_fs_access": make_fs_access,
              "tmpdir": "/tmp",
+             "outdir": "/tmp",
              "enable_reuse": enable_reuse,
              "priority": 500,
-             "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
+             "project_uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz",
+             "workflow_eval_lock": threading.Condition(threading.RLock())
             })
 
         if isinstance(runner, mock.MagicMock):
@@ -1053,68 +1058,90 @@ class TestContainer(unittest.TestCase):
         runner.api.collections().get().execute.return_value = {
             "portable_data_hash": "99999999999999999999999999999993+99"}
 
-        tool = cmap({
-            "inputs": [],
-            "outputs": [],
-            "baseCommand": "nvidia-smi",
-            "arguments": [],
-            "id": "",
-            "cwlVersion": "v1.2",
-            "class": "CommandLineTool",
-            "requirements": [
-            {
+        test_cwl_req = [{
                 "class": "http://commonwl.org/cwltool#CUDARequirement",
                 "cudaVersionMin": "11.0",
-                "cudaComputeCapabilityMin": "9.0",
-            }
-        ]
-        })
+                "cudaComputeCapability": "9.0",
+            }, {
+                "class": "http://commonwl.org/cwltool#CUDARequirement",
+                "cudaVersionMin": "11.0",
+                "cudaComputeCapability": "9.0",
+                "cudaDeviceCountMin": 2
+            }, {
+                "class": "http://commonwl.org/cwltool#CUDARequirement",
+                "cudaVersionMin": "11.0",
+                "cudaComputeCapability": ["4.0", "5.0"],
+                "cudaDeviceCountMin": 2
+            }]
+
+        test_arv_req = [{
+            'device_count': 1,
+            'driver_version': "11.0",
+            'hardware_capability': "9.0"
+        }, {
+            'device_count': 2,
+            'driver_version': "11.0",
+            'hardware_capability': "9.0"
+        }, {
+            'device_count': 2,
+            'driver_version': "11.0",
+            'hardware_capability': "4.0"
+        }]
+
+        for test_case in range(0, len(test_cwl_req)):
 
-        loadingContext, runtimeContext = self.helper(runner, True)
+            tool = cmap({
+                "inputs": [],
+                "outputs": [],
+                "baseCommand": "nvidia-smi",
+                "arguments": [],
+                "id": "",
+                "cwlVersion": "v1.2",
+                "class": "CommandLineTool",
+                "requirements": [test_cwl_req[test_case]]
+            })
 
-        arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
-        arvtool.formatgraph = None
+            loadingContext, runtimeContext = self.helper(runner, True)
 
-        for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
-            j.run(runtimeContext)
-            runner.api.container_requests().create.assert_called_with(
-                body=JsonDiffMatcher({
-                    'environment': {
-                        'HOME': '/var/spool/cwl',
-                        'TMPDIR': '/tmp'
-                    },
-                    'name': 'test_run_True',
-                    'runtime_constraints': {
-                        'vcpus': 1,
-                        'ram': 268435456,
-                        'cuda': {
-                            'device_count': 1,
-                            'driver_version': "11.0",
-                            'hardware_capability': "9.0"
-                        }
-                    },
-                    'use_existing': True,
-                    'priority': 500,
-                    'mounts': {
-                        '/tmp': {'kind': 'tmp',
-                                 "capacity": 1073741824
-                             },
-                        '/var/spool/cwl': {'kind': 'tmp',
-                                           "capacity": 1073741824 }
-                    },
-                    'state': 'Committed',
-                    'output_name': 'Output for step test_run_True',
-                    'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
-                    'output_path': '/var/spool/cwl',
-                    'output_ttl': 0,
-                    'container_image': '99999999999999999999999999999993+99',
-                    'command': ['nvidia-smi'],
-                    'cwd': '/var/spool/cwl',
-                    'scheduling_parameters': {},
-                    'properties': {},
-                    'secret_mounts': {},
-                    'output_storage_classes': ["default"]
-                }))
+            arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+            arvtool.formatgraph = None
+
+            for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+                j.run(runtimeContext)
+                runner.api.container_requests().create.assert_called_with(
+                    body=JsonDiffMatcher({
+                        'environment': {
+                            'HOME': '/var/spool/cwl',
+                            'TMPDIR': '/tmp'
+                        },
+                        'name': 'test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+                        'runtime_constraints': {
+                            'vcpus': 1,
+                            'ram': 268435456,
+                            'cuda': test_arv_req[test_case]
+                        },
+                        'use_existing': True,
+                        'priority': 500,
+                        'mounts': {
+                            '/tmp': {'kind': 'tmp',
+                                     "capacity": 1073741824
+                                 },
+                            '/var/spool/cwl': {'kind': 'tmp',
+                                               "capacity": 1073741824 }
+                        },
+                        'state': 'Committed',
+                        'output_name': 'Output for step test_run_True' + ("" if test_case == 0 else "_"+str(test_case+1)),
+                        'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                        'output_path': '/var/spool/cwl',
+                        'output_ttl': 0,
+                        'container_image': '99999999999999999999999999999993+99',
+                        'command': ['nvidia-smi'],
+                        'cwd': '/var/spool/cwl',
+                        'scheduling_parameters': {},
+                        'properties': {},
+                        'secret_mounts': {},
+                        'output_storage_classes': ["default"]
+                    }))
 
 
     # The test passes no builder.resources
@@ -1148,13 +1175,13 @@ class TestContainer(unittest.TestCase):
             "baseCommand": "echo",
             "arguments": [],
             "id": "",
-            "cwlVersion": "v1.2",
-            "class": "CommandLineTool"
+            "cwlVersion": "v1.0",
+            "class": "org.w3id.cwl.cwl.CommandLineTool"
         })
 
         loadingContext, runtimeContext = self.helper(runner, True)
 
-        arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+        arvtool = arvados_cwl.ArvadosCommandTool(runner, tool, loadingContext)
         arvtool.formatgraph = None
 
         container_request = {
@@ -1165,7 +1192,7 @@ class TestContainer(unittest.TestCase):
             'name': 'test_run_True',
             'runtime_constraints': {
                 'vcpus': 1,
-                'ram': 268435456
+                'ram': 1073741824,
             },
             'use_existing': True,
             'priority': 500,
@@ -1207,6 +1234,103 @@ class TestContainer(unittest.TestCase):
                 body=JsonDiffMatcher(container_request))
 
 
+    # The test passes no builder.resources
+    # Hence the default resources will apply: {'cores': 1, 'ram': 1024, 'outdirSize': 1024, 'tmpdirSize': 1024}
+    @mock.patch("arvados.commands.keepdocker.list_images_in_arv")
+    def test_run_preemptible_hint(self, keepdocker):
+        arvados_cwl.add_arv_hints()
+        for enable_preemptible in (None, True, False):
+            for preemptible_hint in (None, True, False):
+                arv_docker_clear_cache()
+
+                runner = mock.MagicMock()
+                runner.ignore_docker_for_reuse = False
+                runner.intermediate_output_ttl = 0
+                runner.secret_store = cwltool.secrets.SecretStore()
+                runner.api._rootDesc = {"revision": "20210628"}
+
+                keepdocker.return_value = [("zzzzz-4zz18-zzzzzzzzzzzzzz3", "")]
+                runner.api.collections().get().execute.return_value = {
+                    "portable_data_hash": "99999999999999999999999999999993+99"}
+
+                if preemptible_hint is not None:
+                    hints = [{
+                        "class": "http://arvados.org/cwl#UsePreemptible",
+                        "usePreemptible": preemptible_hint
+                    }]
+                else:
+                    hints = []
+
+                tool = cmap({
+                    "inputs": [],
+                    "outputs": [],
+                    "baseCommand": "ls",
+                    "arguments": [{"valueFrom": "$(runtime.outdir)"}],
+                    "id": "",
+                    "class": "CommandLineTool",
+                    "cwlVersion": "v1.2",
+                    "hints": hints
+                })
+
+                loadingContext, runtimeContext = self.helper(runner)
+
+                runtimeContext.name = 'test_run_enable_preemptible_'+str(enable_preemptible)+str(preemptible_hint)
+                runtimeContext.enable_preemptible = enable_preemptible
+
+                arvtool = cwltool.load_tool.load_tool(tool, loadingContext)
+                arvtool.formatgraph = None
+
+                # Test the interactions between --enable/disable-preemptible
+                # and UsePreemptible hint
+
+                if enable_preemptible is None:
+                    if preemptible_hint is None:
+                        sched = {}
+                    else:
+                        sched = {'preemptible': preemptible_hint}
+                else:
+                    if preemptible_hint is None:
+                        sched = {'preemptible': enable_preemptible}
+                    else:
+                        sched = {'preemptible': enable_preemptible and preemptible_hint}
+
+                for j in arvtool.job({}, mock.MagicMock(), runtimeContext):
+                    j.run(runtimeContext)
+                    runner.api.container_requests().create.assert_called_with(
+                        body=JsonDiffMatcher({
+                            'environment': {
+                                'HOME': '/var/spool/cwl',
+                                'TMPDIR': '/tmp'
+                            },
+                            'name': runtimeContext.name,
+                            'runtime_constraints': {
+                                'vcpus': 1,
+                                'ram': 268435456
+                            },
+                            'use_existing': True,
+                            'priority': 500,
+                            'mounts': {
+                                '/tmp': {'kind': 'tmp',
+                                         "capacity": 1073741824
+                                     },
+                                '/var/spool/cwl': {'kind': 'tmp',
+                                                   "capacity": 1073741824 }
+                            },
+                            'state': 'Committed',
+                            'output_name': 'Output for step '+runtimeContext.name,
+                            'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
+                            'output_path': '/var/spool/cwl',
+                            'output_ttl': 0,
+                            'container_image': '99999999999999999999999999999993+99',
+                            'command': ['ls', '/var/spool/cwl'],
+                            'cwd': '/var/spool/cwl',
+                            'scheduling_parameters': sched,
+                            'properties': {},
+                            'secret_mounts': {},
+                            'output_storage_classes': ["default"]
+                        }))
+
+
 
 class TestWorkflow(unittest.TestCase):
     def setUp(self):
diff --git a/sdk/cwl/tests/test_submit.py b/sdk/cwl/tests/test_submit.py
index 10443359b9..61892bf2a4 100644
--- a/sdk/cwl/tests/test_submit.py
+++ b/sdk/cwl/tests/test_submit.py
@@ -1468,6 +1468,49 @@ class TestSubmit(unittest.TestCase):
         self.assertEqual(exited, 0)
 
 
+    @stubs
+    def test_submit_enable_preemptible(self, stubs):
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--enable-preemptible",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
+                        '--no-log-timestamps', '--disable-validate', '--disable-color',
+                        '--eval-timeout=20', '--thread-count=0',
+                        '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
+                                       '--enable-preemptible',
+                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
+
+    @stubs
+    def test_submit_disable_preemptible(self, stubs):
+        exited = arvados_cwl.main(
+            ["--submit", "--no-wait", "--api=containers", "--debug", "--disable-preemptible",
+                "tests/wf/submit_wf.cwl", "tests/submit_test_job.json"],
+            stubs.capture_stdout, sys.stderr, api_client=stubs.api, keep_client=stubs.keep_client)
+
+        expect_container = copy.deepcopy(stubs.expect_container_spec)
+        expect_container['command'] = ['arvados-cwl-runner', '--local', '--api=containers',
+                        '--no-log-timestamps', '--disable-validate', '--disable-color',
+                        '--eval-timeout=20', '--thread-count=0',
+                        '--enable-reuse', "--collection-cache-size=256", '--debug', '--on-error=continue',
+                                       '--disable-preemptible',
+                        '/var/lib/cwl/workflow.json#main', '/var/lib/cwl/cwl.input.json']
+
+        stubs.api.container_requests().create.assert_called_with(
+            body=JsonDiffMatcher(expect_container))
+        self.assertEqual(stubs.capture_stdout.getvalue(),
+                         stubs.expect_container_request_uuid + '\n')
+        self.assertEqual(exited, 0)
+
+
 class TestCreateWorkflow(unittest.TestCase):
     existing_workflow_uuid = "zzzzz-7fd4e-validworkfloyml"
     expect_workflow = StripYAMLComments(
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 7409b18132..d76ece1edd 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -139,6 +139,8 @@ type CreateOptions struct {
 	EnsureUniqueName bool                   `json:"ensure_unique_name"`
 	Select           []string               `json:"select"`
 	Attrs            map[string]interface{} `json:"attrs"`
+	// ReplaceFiles only applies when creating a collection.
+	ReplaceFiles map[string]string `json:"replace_files"`
 }
 
 type UpdateOptions struct {
@@ -146,6 +148,8 @@ type UpdateOptions struct {
 	Attrs            map[string]interface{} `json:"attrs"`
 	Select           []string               `json:"select"`
 	BypassFederation bool                   `json:"bypass_federation"`
+	// ReplaceFiles only applies when updating a collection.
+	ReplaceFiles map[string]string `json:"replace_files"`
 }
 
 type GroupContentsOptions struct {
diff --git a/sdk/go/arvados/fs_backend.go b/sdk/go/arvados/fs_backend.go
index 32365a5317..cc4c32ffe9 100644
--- a/sdk/go/arvados/fs_backend.go
+++ b/sdk/go/arvados/fs_backend.go
@@ -6,6 +6,7 @@ package arvados
 
 import (
 	"context"
+	"errors"
 	"io"
 )
 
@@ -30,3 +31,16 @@ type keepClient interface {
 type apiClient interface {
 	RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error
 }
+
+var errStubClient = errors.New("stub client")
+
+type StubClient struct{}
+
+func (*StubClient) ReadAt(string, []byte, int) (int, error) { return 0, errStubClient }
+func (*StubClient) LocalLocator(loc string) (string, error) { return loc, nil }
+func (*StubClient) BlockWrite(context.Context, BlockWriteOptions) (BlockWriteResponse, error) {
+	return BlockWriteResponse{}, errStubClient
+}
+func (*StubClient) RequestAndDecode(_ interface{}, _, _ string, _ io.Reader, _ interface{}) error {
+	return errStubClient
+}
diff --git a/sdk/go/arvados/fs_base.go b/sdk/go/arvados/fs_base.go
index 80b8037293..bebb74261e 100644
--- a/sdk/go/arvados/fs_base.go
+++ b/sdk/go/arvados/fs_base.go
@@ -8,6 +8,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"log"
 	"net/http"
 	"os"
@@ -159,6 +160,18 @@ type FileSystem interface {
 	MemorySize() int64
 }
 
+type fsFS struct {
+	FileSystem
+}
+
+// FS returns an fs.FS interface to the given FileSystem, to enable
+// the use of fs.WalkDir, etc.
+func FS(fs FileSystem) fs.FS { return fsFS{fs} }
+func (fs fsFS) Open(path string) (fs.File, error) {
+	f, err := fs.FileSystem.Open(path)
+	return f, err
+}
+
 type inode interface {
 	SetParent(parent inode, name string)
 	Parent() inode
@@ -450,14 +463,14 @@ func (fs *fileSystem) openFile(name string, flag int, perm os.FileMode) (*fileha
 	default:
 		return nil, fmt.Errorf("invalid flags 0x%x", flag)
 	}
-	if !writable && parent.IsDir() {
+	if parent.IsDir() {
 		// A directory can be opened via "foo/", "foo/.", or
 		// "foo/..".
 		switch name {
 		case ".", "":
-			return &filehandle{inode: parent}, nil
+			return &filehandle{inode: parent, readable: readable, writable: writable}, nil
 		case "..":
-			return &filehandle{inode: parent.Parent()}, nil
+			return &filehandle{inode: parent.Parent(), readable: readable, writable: writable}, nil
 		}
 	}
 	createMode := flag&os.O_CREATE != 0
@@ -753,7 +766,7 @@ func Splice(fs FileSystem, target string, newsubtree *Subtree) error {
 		f, err = fs.OpenFile(target, os.O_CREATE|os.O_WRONLY, 0700)
 	}
 	if err != nil {
-		return err
+		return fmt.Errorf("open %s: %w", target, err)
 	}
 	defer f.Close()
 	return f.Splice(newsubtree)
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 0c5819721e..f4dae746e2 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -1565,7 +1565,7 @@ func (dn *dirnode) snapshot() (*dirnode, error) {
 func (dn *dirnode) Splice(repl inode) error {
 	repl, err := repl.Snapshot()
 	if err != nil {
-		return err
+		return fmt.Errorf("cannot copy snapshot: %w", err)
 	}
 	switch repl := repl.(type) {
 	default:
@@ -1599,7 +1599,7 @@ func (dn *dirnode) Splice(repl inode) error {
 		defer dn.Unlock()
 		_, err = dn.parent.Child(dn.fileinfo.name, func(inode) (inode, error) { return repl, nil })
 		if err != nil {
-			return err
+			return fmt.Errorf("error replacing filenode: dn.parent.Child(): %w", err)
 		}
 		repl.fs = dn.fs
 	}
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index fab91d1f77..b221aaa083 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -1441,6 +1441,30 @@ func (s *CollectionFSSuite) TestEdgeCaseManifests(c *check.C) {
 	}
 }
 
+func (s *CollectionFSSuite) TestSnapshotSplice(c *check.C) {
+	filedata1 := "hello snapshot+splice world\n"
+	fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+	c.Assert(err, check.IsNil)
+	{
+		f, err := fs.OpenFile("file1", os.O_CREATE|os.O_RDWR, 0700)
+		c.Assert(err, check.IsNil)
+		_, err = f.Write([]byte(filedata1))
+		c.Assert(err, check.IsNil)
+		err = f.Close()
+		c.Assert(err, check.IsNil)
+	}
+
+	snap, err := Snapshot(fs, "/")
+	c.Assert(err, check.IsNil)
+	err = Splice(fs, "dir1", snap)
+	c.Assert(err, check.IsNil)
+	f, err := fs.Open("dir1/file1")
+	c.Assert(err, check.IsNil)
+	buf, err := io.ReadAll(f)
+	c.Assert(err, check.IsNil)
+	c.Check(string(buf), check.Equals, filedata1)
+}
+
 func (s *CollectionFSSuite) TestRefreshSignatures(c *check.C) {
 	filedata1 := "hello refresh signatures world\n"
 	fs, err := (&Collection{}).FileSystem(s.client, s.kc)
diff --git a/sdk/go/arvados/fs_filehandle.go b/sdk/go/arvados/fs_filehandle.go
index 4530a7b06a..f50dd4612b 100644
--- a/sdk/go/arvados/fs_filehandle.go
+++ b/sdk/go/arvados/fs_filehandle.go
@@ -6,6 +6,7 @@ package arvados
 
 import (
 	"io"
+	"io/fs"
 	"os"
 )
 
@@ -73,6 +74,31 @@ func (f *filehandle) Write(p []byte) (n int, err error) {
 	return
 }
 
+// dirEntry implements fs.DirEntry, see (*filehandle)ReadDir().
+type dirEntry struct {
+	os.FileInfo
+}
+
+func (ent dirEntry) Type() fs.FileMode {
+	return ent.Mode().Type()
+}
+func (ent dirEntry) Info() (fs.FileInfo, error) {
+	return ent, nil
+}
+
+// ReadDir implements fs.ReadDirFile.
+func (f *filehandle) ReadDir(count int) ([]fs.DirEntry, error) {
+	fis, err := f.Readdir(count)
+	if len(fis) == 0 {
+		return nil, err
+	}
+	ents := make([]fs.DirEntry, len(fis))
+	for i, fi := range fis {
+		ents[i] = dirEntry{fi}
+	}
+	return ents, err
+}
+
 func (f *filehandle) Readdir(count int) ([]os.FileInfo, error) {
 	if !f.inode.IsDir() {
 		return nil, ErrInvalidOperation
diff --git a/sdk/go/arvados/fs_site_test.go b/sdk/go/arvados/fs_site_test.go
index 59fa5fc176..bf24efa7ed 100644
--- a/sdk/go/arvados/fs_site_test.go
+++ b/sdk/go/arvados/fs_site_test.go
@@ -10,6 +10,7 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"strings"
 	"syscall"
 	"time"
 
@@ -291,40 +292,41 @@ func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 		c.Check(string(buf), check.Equals, string(thisfile))
 	}
 
-	// Cannot splice a file onto a collection root, or anywhere
-	// outside a collection
+	// Cannot splice a file onto a collection root; cannot splice
+	// anything to a target outside a collection.
 	for _, badpath := range []string{
+		dstPath + "/",
 		dstPath,
+		"/home/A Project/newnodename/",
 		"/home/A Project/newnodename",
+		"/home/A Project/",
 		"/home/A Project",
+		"/home/newnodename/",
 		"/home/newnodename",
+		"/home/",
 		"/home",
+		"/newnodename/",
 		"/newnodename",
+		"/",
 	} {
 		err = Splice(s.fs, badpath, snapFile)
 		c.Check(err, check.NotNil)
-		c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
-		if badpath == dstPath {
-			c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %s", badpath))
+		if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
+			c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
+		} else {
+			c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
+		}
+		if strings.TrimSuffix(badpath, "/") == dstPath {
+			c.Check(err, check.ErrorMatches, `cannot use Splice to attach a file at top level of \*arvados.collectionFileSystem: invalid operation`, check.Commentf("badpath: %q", badpath))
 			continue
 		}
-		err = Splice(s.fs, badpath, snap1)
-		c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %s"))
-	}
 
-	// Destination cannot have trailing slash
-	for _, badpath := range []string{
-		dstPath + "/ctxlog/",
-		dstPath + "/",
-		"/home/A Project/",
-		"/home/",
-		"/",
-		"",
-	} {
 		err = Splice(s.fs, badpath, snap1)
-		c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
-		err = Splice(s.fs, badpath, snapFile)
-		c.Check(err, ErrorIs, ErrInvalidArgument, check.Commentf("badpath %s", badpath))
+		if strings.Contains(badpath, "newnodename") && strings.HasSuffix(badpath, "/") {
+			c.Check(err, ErrorIs, os.ErrNotExist, check.Commentf("badpath %q", badpath))
+		} else {
+			c.Check(err, ErrorIs, ErrInvalidOperation, check.Commentf("badpath %q", badpath))
+		}
 	}
 
 	// Destination's parent must already exist
@@ -340,9 +342,10 @@ func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 	}
 
 	snap2, err := Snapshot(s.fs, dstPath+"/ctxlog-copy")
-	c.Check(err, check.IsNil)
-	err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
-	c.Check(err, check.IsNil)
+	if c.Check(err, check.IsNil) {
+		err = Splice(s.fs, dstPath+"/ctxlog-copy-copy", snap2)
+		c.Check(err, check.IsNil)
+	}
 
 	// Snapshot entire collection, splice into same collection at
 	// a new path, remove file from original location, verify
@@ -362,9 +365,10 @@ func (s *SiteFSSuite) TestSnapshotSplice(c *check.C) {
 	_, err = s.fs.Open(dstPath + "/arvados/fs_site_test.go")
 	c.Check(err, check.Equals, os.ErrNotExist)
 	f, err = s.fs.Open(dstPath + "/copy2/arvados/fs_site_test.go")
-	c.Check(err, check.IsNil)
-	defer f.Close()
-	buf, err := ioutil.ReadAll(f)
-	c.Check(err, check.IsNil)
-	c.Check(string(buf), check.Equals, string(thisfile))
+	if c.Check(err, check.IsNil) {
+		defer f.Close()
+		buf, err := ioutil.ReadAll(f)
+		c.Check(err, check.IsNil)
+		c.Check(string(buf), check.Equals, string(thisfile))
+	}
 }
diff --git a/sdk/go/arvadostest/oidc_provider.go b/sdk/go/arvadostest/oidc_provider.go
index fa5e55c42e..087adc4b24 100644
--- a/sdk/go/arvadostest/oidc_provider.go
+++ b/sdk/go/arvadostest/oidc_provider.go
@@ -35,6 +35,12 @@ type OIDCProvider struct {
 
 	PeopleAPIResponse map[string]interface{}
 
+	// send incoming /userinfo requests to HoldUserInfo (if not
+	// nil), then receive from ReleaseUserInfo (if not nil),
+	// before responding (these are used to set up races)
+	HoldUserInfo    chan *http.Request
+	ReleaseUserInfo chan struct{}
+
 	key       *rsa.PrivateKey
 	Issuer    *httptest.Server
 	PeopleAPI *httptest.Server
@@ -126,6 +132,12 @@ func (p *OIDCProvider) serveOIDC(w http.ResponseWriter, req *http.Request) {
 	case "/auth":
 		w.WriteHeader(http.StatusInternalServerError)
 	case "/userinfo":
+		if p.HoldUserInfo != nil {
+			p.HoldUserInfo <- req
+		}
+		if p.ReleaseUserInfo != nil {
+			<-p.ReleaseUserInfo
+		}
 		authhdr := req.Header.Get("Authorization")
 		if _, err := jwt.ParseSigned(strings.TrimPrefix(authhdr, "Bearer ")); err != nil {
 			p.c.Logf("OIDCProvider: bad auth %q", authhdr)
diff --git a/sdk/go/httpserver/error.go b/sdk/go/httpserver/error.go
index f1817d3374..75ff85336f 100644
--- a/sdk/go/httpserver/error.go
+++ b/sdk/go/httpserver/error.go
@@ -6,9 +6,14 @@ package httpserver
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 )
 
+func Errorf(status int, tmpl string, args ...interface{}) error {
+	return errorWithStatus{fmt.Errorf(tmpl, args...), status}
+}
+
 func ErrorWithStatus(err error, status int) error {
 	return errorWithStatus{err, status}
 }
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index 88596211d4..e0d1c50f03 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -253,6 +253,7 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
     svc.insecure = insecure
     svc.request_id = request_id
     svc.config = lambda: util.get_config_once(svc)
+    svc.vocabulary = lambda: util.get_vocabulary_once(svc)
     kwargs['http'].max_request_size = svc._rootDesc.get('maxRequestSize', 0)
     kwargs['http'].cache = None
     kwargs['http']._request_id = lambda: svc.request_id or util.new_request_id()
diff --git a/sdk/python/arvados/commands/run.py b/sdk/python/arvados/commands/run.py
index 1e64eeb1da..0fe05da22b 100644
--- a/sdk/python/arvados/commands/run.py
+++ b/sdk/python/arvados/commands/run.py
@@ -65,7 +65,7 @@ def is_in_collection(root, branch):
             return (None, None)
         fn = os.path.join(root, ".arvados#collection")
         if os.path.exists(fn):
-            with file(fn, 'r') as f:
+            with open(fn, 'r') as f:
                 c = json.load(f)
             return (c["portable_data_hash"], branch)
         else:
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 0018687ff3..1a83eae944 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -376,6 +376,7 @@ class KeepClient(object):
                     curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
                     if self.insecure:
                         curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+                        curl.setopt(pycurl.SSL_VERIFYHOST, 0)
                     else:
                         curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
                     if method == "HEAD":
@@ -478,6 +479,7 @@ class KeepClient(object):
                     curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
                     if self.insecure:
                         curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+                        curl.setopt(pycurl.SSL_VERIFYHOST, 0)
                     else:
                         curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
                     self._setcurltimeouts(curl, timeout)
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 2380e48b73..be8a03fc31 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -491,3 +491,11 @@ def get_config_once(svc):
     if not hasattr(svc, '_cached_config'):
         svc._cached_config = svc.configs().get().execute()
     return svc._cached_config
+
+def get_vocabulary_once(svc):
+    if not svc._rootDesc.get('resources').get('vocabularies', False):
+        # Old API server version, no vocabulary export endpoint
+        return {}
+    if not hasattr(svc, '_cached_vocabulary'):
+        svc._cached_vocabulary = svc.vocabularies().get().execute()
+    return svc._cached_vocabulary
diff --git a/sdk/python/arvados/vocabulary.py b/sdk/python/arvados/vocabulary.py
new file mode 100644
index 0000000000..3bb87c48dc
--- /dev/null
+++ b/sdk/python/arvados/vocabulary.py
@@ -0,0 +1,127 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+
+from . import api
+
+_logger = logging.getLogger('arvados.vocabulary')
+
+def load_vocabulary(api_client=None):
+    """Load the Arvados vocabulary from the API.
+    """
+    if api_client is None:
+        api_client = api('v1')
+    return Vocabulary(api_client.vocabulary())
+
+class VocabularyError(Exception):
+    """Base class for all vocabulary errors.
+    """
+    pass
+
+class VocabularyKeyError(VocabularyError):
+    pass
+
+class VocabularyValueError(VocabularyError):
+    pass
+
+class Vocabulary(object):
+    def __init__(self, voc_definition={}):
+        self.strict_keys = voc_definition.get('strict_tags', False)
+        self.key_aliases = {}
+
+        for key_id, val in (voc_definition.get('tags') or {}).items():
+            strict = val.get('strict', False)
+            key_labels = [l['label'] for l in val.get('labels', [])]
+            values = {}
+            for v_id, v_val in (val.get('values') or {}).items():
+                labels = [l['label'] for l in v_val.get('labels', [])]
+                values[v_id] = VocabularyValue(v_id, labels)
+            vk = VocabularyKey(key_id, key_labels, values, strict)
+            self.key_aliases[key_id.lower()] = vk
+            for alias in vk.aliases:
+                self.key_aliases[alias.lower()] = vk
+
+    def __getitem__(self, key):
+        return self.key_aliases[key.lower()]
+
+    def convert_to_identifiers(self, obj={}):
+        """Translate key/value pairs to machine readable identifiers.
+        """
+        return self._convert_to_what(obj, 'identifier')
+
+    def convert_to_labels(self, obj={}):
+        """Translate key/value pairs to human readable labels.
+        """
+        return self._convert_to_what(obj, 'preferred_label')
+
+    def _convert_to_what(self, obj={}, what=None):
+        if not isinstance(obj, dict):
+            raise ValueError("obj must be a dict")
+        if what not in ['preferred_label', 'identifier']:
+            raise ValueError("what attr must be 'preferred_label' or 'identifier'")
+        r = {}
+        for k, v in obj.items():
+            # Key validation & lookup
+            key_found = False
+            if not isinstance(k, str):
+                raise VocabularyKeyError("key '{}' must be a string".format(k))
+            k_what, v_what = k, v
+            try:
+                k_what = getattr(self[k], what)
+                key_found = True
+            except KeyError:
+                if self.strict_keys:
+                    raise VocabularyKeyError("key '{}' not found in vocabulary".format(k))
+
+            # Value validation & lookup
+            if isinstance(v, list):
+                v_what = []
+                for x in v:
+                    if not isinstance(x, str):
+                        raise VocabularyValueError("value '{}' for key '{}' must be a string".format(x, k))
+                    try:
+                        v_what.append(getattr(self[k][x], what))
+                    except KeyError:
+                        if self[k].strict:
+                            raise VocabularyValueError("value '{}' not found for key '{}'".format(x, k))
+                        v_what.append(x)
+            else:
+                if not isinstance(v, str):
+                    raise VocabularyValueError("{} value '{}' for key '{}' must be a string".format(type(v).__name__, v, k))
+                try:
+                    v_what = getattr(self[k][v], what)
+                except KeyError:
+                    if key_found and self[k].strict:
+                        raise VocabularyValueError("value '{}' not found for key '{}'".format(v, k))
+
+            r[k_what] = v_what
+        return r
+
+class VocabularyData(object):
+    def __init__(self, identifier, aliases=[]):
+        self.identifier = identifier
+        self.aliases = aliases
+
+    def __getattribute__(self, name):
+        if name == 'preferred_label':
+            return self.aliases[0]
+        return super(VocabularyData, self).__getattribute__(name)
+
+class VocabularyValue(VocabularyData):
+    def __init__(self, identifier, aliases=[]):
+        super(VocabularyValue, self).__init__(identifier, aliases)
+
+class VocabularyKey(VocabularyData):
+    def __init__(self, identifier, aliases=[], values={}, strict=False):
+        super(VocabularyKey, self).__init__(identifier, aliases)
+        self.strict = strict
+        self.value_aliases = {}
+        for v_id, v_val in values.items():
+            self.value_aliases[v_id.lower()] = v_val
+            for v_alias in v_val.aliases:
+                self.value_aliases[v_alias.lower()] = v_val
+
+    def __getitem__(self, key):
+        return self.value_aliases[key.lower()]
\ No newline at end of file
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index f82d44ab60..126b12f156 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -51,7 +51,7 @@ setup(name='arvados-python-client',
           'google-api-python-client >=1.6.2, <2',
           'google-auth<2',
           'httplib2 >=0.9.2, <0.20.2',
-          'pycurl >=7.19.5.1',
+          'pycurl >=7.19.5.1, <7.45.0',
           'ruamel.yaml >=0.15.54, <0.17.11',
           'setuptools',
           'ws4py >=0.4.2',
diff --git a/sdk/python/tests/test_keep_client.py b/sdk/python/tests/test_keep_client.py
index aa7e371bf4..605b90301c 100644
--- a/sdk/python/tests/test_keep_client.py
+++ b/sdk/python/tests/test_keep_client.py
@@ -265,6 +265,9 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
             self.assertEqual(
                 mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
                 0)
+            self.assertEqual(
+                mock.responses[0].getopt(pycurl.SSL_VERIFYHOST),
+                0)
 
         api_client.insecure = False
         with tutil.mock_keep_responses(b'foo', 200) as mock:
@@ -276,6 +279,9 @@ class KeepClientServiceTestCase(unittest.TestCase, tutil.ApiClientMock):
             self.assertEqual(
                 mock.responses[0].getopt(pycurl.SSL_VERIFYPEER),
                 None)
+            self.assertEqual(
+                mock.responses[0].getopt(pycurl.SSL_VERIFYHOST),
+                None)
 
     def test_refresh_signature(self):
         blk_digest = '6f5902ac237024bdd0c176cb93063dc4+11'
diff --git a/sdk/python/tests/test_vocabulary.py b/sdk/python/tests/test_vocabulary.py
new file mode 100644
index 0000000000..aa2e739e20
--- /dev/null
+++ b/sdk/python/tests/test_vocabulary.py
@@ -0,0 +1,319 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import arvados
+import unittest
+import mock
+
+from arvados import api, vocabulary
+
+class VocabularyTest(unittest.TestCase):
+    EXAMPLE_VOC = {
+        'tags': {
+            'IDTAGANIMALS': {
+                'strict': False,
+                'labels': [
+                    {'label': 'Animal'},
+                    {'label': 'Creature'},
+                ],
+                'values': {
+                    'IDVALANIMAL1': {
+                        'labels': [
+                            {'label': 'Human'},
+                            {'label': 'Homo sapiens'},
+                        ],
+                    },
+                    'IDVALANIMAL2': {
+                        'labels': [
+                            {'label': 'Elephant'},
+                            {'label': 'Loxodonta'},
+                        ],
+                    },
+                },
+            },
+            'IDTAGIMPORTANCES': {
+                'strict': True,
+                'labels': [
+                    {'label': 'Importance'},
+                    {'label': 'Priority'},
+                ],
+                'values': {
+                    'IDVALIMPORTANCE1': {
+                        'labels': [
+                            {'label': 'High'},
+                            {'label': 'High priority'},
+                        ],
+                    },
+                    'IDVALIMPORTANCE2': {
+                        'labels': [
+                            {'label': 'Medium'},
+                            {'label': 'Medium priority'},
+                        ],
+                    },
+                    'IDVALIMPORTANCE3': {
+                        'labels': [
+                            {'label': 'Low'},
+                            {'label': 'Low priority'},
+                        ],
+                    },
+                },
+            },
+            'IDTAGCOMMENTS': {
+                'strict': False,
+                'labels': [
+                    {'label': 'Comment'},
+                    {'label': 'Notes'},
+                ],
+                'values': None,
+            },
+        },
+    }
+
+    def setUp(self):
+        self.api = arvados.api('v1')
+        self.voc = vocabulary.Vocabulary(self.EXAMPLE_VOC)
+        self.api.vocabulary = mock.MagicMock(return_value=self.EXAMPLE_VOC)
+
+    def test_vocabulary_keys(self):
+        self.assertEqual(self.voc.strict_keys, False)
+        self.assertEqual(
+            self.voc.key_aliases.keys(),
+            set(['idtaganimals', 'creature', 'animal',
+                'idtagimportances', 'importance', 'priority',
+                'idtagcomments', 'comment', 'notes'])
+        )
+
+        vk = self.voc.key_aliases['creature']
+        self.assertEqual(vk.strict, False)
+        self.assertEqual(vk.identifier, 'IDTAGANIMALS')
+        self.assertEqual(vk.aliases, ['Animal', 'Creature'])
+        self.assertEqual(vk.preferred_label, 'Animal')
+        self.assertEqual(
+            vk.value_aliases.keys(),
+            set(['idvalanimal1', 'human', 'homo sapiens',
+                'idvalanimal2', 'elephant', 'loxodonta'])
+        )
+
+    def test_vocabulary_values(self):
+        vk = self.voc.key_aliases['creature']
+        vv = vk.value_aliases['human']
+        self.assertEqual(vv.identifier, 'IDVALANIMAL1')
+        self.assertEqual(vv.aliases, ['Human', 'Homo sapiens'])
+        self.assertEqual(vv.preferred_label, 'Human')
+
+    def test_vocabulary_indexing(self):
+        self.assertEqual(self.voc['creature']['human'].identifier, 'IDVALANIMAL1')
+        self.assertEqual(self.voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+        self.assertEqual(self.voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
+        with self.assertRaises(KeyError):
+            inexistant = self.voc['foo']
+
+    def test_empty_vocabulary(self):
+        voc = vocabulary.Vocabulary({})
+        self.assertEqual(voc.strict_keys, False)
+        self.assertEqual(voc.key_aliases, {})
+
+    def test_load_vocabulary_with_api(self):
+        voc = vocabulary.load_vocabulary(self.api)
+        self.assertEqual(voc['creature']['human'].identifier, 'IDVALANIMAL1')
+        self.assertEqual(voc['Creature']['Human'].identifier, 'IDVALANIMAL1')
+        self.assertEqual(voc['CREATURE']['HUMAN'].identifier, 'IDVALANIMAL1')
+
+    def test_convert_to_identifiers(self):
+        cases = [
+            {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+            {'IDTAGIMPORTANCES': 'High'},
+            {'importance': 'IDVALIMPORTANCE1'},
+            {'priority': 'high priority'},
+        ]
+        for case in cases:
+            self.assertEqual(
+                self.voc.convert_to_identifiers(case),
+                {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+                "failing test case: {}".format(case)
+            )
+
+    def test_convert_to_identifiers_multiple_pairs(self):
+        cases = [
+            {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+            {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+            {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+            {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
+        ]
+        for case in cases:
+            self.assertEqual(
+                self.voc.convert_to_identifiers(case),
+                {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+                "failing test case: {}".format(case)
+            )
+
+    def test_convert_to_identifiers_value_lists(self):
+        cases = [
+            {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+            {'IDTAGIMPORTANCES': ['High', 'Medium']},
+            {'importance': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+            {'priority': ['high', 'medium']},
+        ]
+        for case in cases:
+            self.assertEqual(
+                self.voc.convert_to_identifiers(case),
+                {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+                "failing test case: {}".format(case)
+            )
+
+    def test_convert_to_identifiers_unknown_key(self):
+        # Non-strict vocabulary
+        self.assertEqual(self.voc.strict_keys, False)
+        self.assertEqual(self.voc.convert_to_identifiers({'foo': 'bar'}), {'foo': 'bar'})
+        # Strict vocabulary
+        strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+        strict_voc.strict_keys = True
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            strict_voc.convert_to_identifiers({'foo': 'bar'})
+
+    def test_convert_to_identifiers_invalid_key(self):
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            self.voc.convert_to_identifiers({42: 'bar'})
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            self.voc.convert_to_identifiers({None: 'bar'})
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            self.voc.convert_to_identifiers({('f', 'o', 'o'): 'bar'})
+
+    def test_convert_to_identifiers_unknown_value(self):
+        # Non-strict key
+        self.assertEqual(self.voc['animal'].strict, False)
+        self.assertEqual(self.voc.convert_to_identifiers({'Animal': 'foo'}), {'IDTAGANIMALS': 'foo'})
+        # Strict key
+        self.assertEqual(self.voc['priority'].strict, True)
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Priority': 'foo'})
+
+    def test_convert_to_identifiers_invalid_value(self):
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Animal': 42})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Animal': None})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Animal': {'hello': 'world'}})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Animal': [42]})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Animal': [None]})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Animal': [{'hello': 'world'}]})
+
+    def test_convert_to_identifiers_unknown_value_list(self):
+        # Non-strict key
+        self.assertEqual(self.voc['animal'].strict, False)
+        self.assertEqual(
+            self.voc.convert_to_identifiers({'Animal': ['foo', 'loxodonta']}),
+            {'IDTAGANIMALS': ['foo', 'IDVALANIMAL2']}
+        )
+        # Strict key
+        self.assertEqual(self.voc['priority'].strict, True)
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_identifiers({'Priority': ['foo', 'bar']})
+
+    def test_convert_to_labels(self):
+        cases = [
+            {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1'},
+            {'IDTAGIMPORTANCES': 'High'},
+            {'importance': 'IDVALIMPORTANCE1'},
+            {'priority': 'high priority'},
+        ]
+        for case in cases:
+            self.assertEqual(
+                self.voc.convert_to_labels(case),
+                {'Importance': 'High'},
+                "failing test case: {}".format(case)
+            )
+
+    def test_convert_to_labels_multiple_pairs(self):
+        cases = [
+            {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'},
+            {'IDTAGIMPORTANCES': 'High', 'IDTAGANIMALS': 'IDVALANIMAL1', 'comment': 'Very important person'},
+            {'importance': 'IDVALIMPORTANCE1', 'animal': 'IDVALANIMAL1', 'notes': 'Very important person'},
+            {'priority': 'high priority', 'animal': 'IDVALANIMAL1', 'NOTES': 'Very important person'},
+        ]
+        for case in cases:
+            self.assertEqual(
+                self.voc.convert_to_labels(case),
+                {'Importance': 'High', 'Animal': 'Human', 'Comment': 'Very important person'},
+                "failing test case: {}".format(case)
+            )
+
+    def test_convert_to_labels_value_lists(self):
+        cases = [
+            {'IDTAGIMPORTANCES': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+            {'IDTAGIMPORTANCES': ['High', 'Medium']},
+            {'importance': ['IDVALIMPORTANCE1', 'IDVALIMPORTANCE2']},
+            {'priority': ['high', 'medium']},
+        ]
+        for case in cases:
+            self.assertEqual(
+                self.voc.convert_to_labels(case),
+                {'Importance': ['High', 'Medium']},
+                "failing test case: {}".format(case)
+            )
+
+    def test_convert_to_labels_unknown_key(self):
+        # Non-strict vocabulary
+        self.assertEqual(self.voc.strict_keys, False)
+        self.assertEqual(self.voc.convert_to_labels({'foo': 'bar'}), {'foo': 'bar'})
+        # Strict vocabulary
+        strict_voc = arvados.vocabulary.Vocabulary(self.EXAMPLE_VOC)
+        strict_voc.strict_keys = True
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            strict_voc.convert_to_labels({'foo': 'bar'})
+
+    def test_convert_to_labels_invalid_key(self):
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            self.voc.convert_to_labels({42: 'bar'})
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            self.voc.convert_to_labels({None: 'bar'})
+        with self.assertRaises(vocabulary.VocabularyKeyError):
+            self.voc.convert_to_labels({('f', 'o', 'o'): 'bar'})
+
+    def test_convert_to_labels_unknown_value(self):
+        # Non-strict key
+        self.assertEqual(self.voc['animal'].strict, False)
+        self.assertEqual(self.voc.convert_to_labels({'IDTAGANIMALS': 'foo'}), {'Animal': 'foo'})
+        # Strict key
+        self.assertEqual(self.voc['priority'].strict, True)
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': 'foo'})
+
+    def test_convert_to_labels_invalid_value(self):
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': {'high': True}})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': None})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': 42})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': False})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': [42]})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': [None]})
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': [{'high': True}]})
+
+    def test_convert_to_labels_unknown_value_list(self):
+        # Non-strict key
+        self.assertEqual(self.voc['animal'].strict, False)
+        self.assertEqual(
+            self.voc.convert_to_labels({'IDTAGANIMALS': ['foo', 'IDVALANIMAL1']}),
+            {'Animal': ['foo', 'Human']}
+        )
+        # Strict key
+        self.assertEqual(self.voc['priority'].strict, True)
+        with self.assertRaises(vocabulary.VocabularyValueError):
+            self.voc.convert_to_labels({'IDTAGIMPORTANCES': ['foo', 'bar']})
+
+    def test_convert_roundtrip(self):
+        initial = {'IDTAGIMPORTANCES': 'IDVALIMPORTANCE1', 'IDTAGANIMALS': 'IDVALANIMAL1', 'IDTAGCOMMENTS': 'Very important person'}
+        converted = self.voc.convert_to_labels(initial)
+        self.assertNotEqual(converted, initial)
+        self.assertEqual(self.voc.convert_to_identifiers(converted), initial)
diff --git a/services/api/Gemfile b/services/api/Gemfile
index ac6bfdb01f..30d877fac7 100644
--- a/services/api/Gemfile
+++ b/services/api/Gemfile
@@ -32,8 +32,6 @@ gem 'oj'
 
 gem 'jquery-rails'
 
-gem 'rvm-capistrano', :group => :test
-
 gem 'acts_as_api'
 
 gem 'passenger'
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 74f26e4d15..c5ecbaef7a 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -8,43 +8,43 @@ GIT
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.6)
-      actionpack (= 5.2.6)
+    actioncable (5.2.6.3)
+      actionpack (= 5.2.6.3)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.6)
-      actionpack (= 5.2.6)
-      actionview (= 5.2.6)
-      activejob (= 5.2.6)
+    actionmailer (5.2.6.3)
+      actionpack (= 5.2.6.3)
+      actionview (= 5.2.6.3)
+      activejob (= 5.2.6.3)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.6)
-      actionview (= 5.2.6)
-      activesupport (= 5.2.6)
+    actionpack (5.2.6.3)
+      actionview (= 5.2.6.3)
+      activesupport (= 5.2.6.3)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.6)
-      activesupport (= 5.2.6)
+    actionview (5.2.6.3)
+      activesupport (= 5.2.6.3)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
       rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.6)
-      activesupport (= 5.2.6)
+    activejob (5.2.6.3)
+      activesupport (= 5.2.6.3)
       globalid (>= 0.3.6)
-    activemodel (5.2.6)
-      activesupport (= 5.2.6)
-    activerecord (5.2.6)
-      activemodel (= 5.2.6)
-      activesupport (= 5.2.6)
+    activemodel (5.2.6.3)
+      activesupport (= 5.2.6.3)
+    activerecord (5.2.6.3)
+      activemodel (= 5.2.6.3)
+      activesupport (= 5.2.6.3)
       arel (>= 9.0)
-    activestorage (5.2.6)
-      actionpack (= 5.2.6)
-      activerecord (= 5.2.6)
+    activestorage (5.2.6.3)
+      actionpack (= 5.2.6.3)
+      activerecord (= 5.2.6.3)
       marcel (~> 1.0.0)
-    activesupport (5.2.6)
+    activesupport (5.2.6.3)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
@@ -82,12 +82,6 @@ GEM
       multi_json (>= 1.0.0)
     builder (3.2.4)
     byebug (11.0.1)
-    capistrano (2.15.9)
-      highline
-      net-scp (>= 1.0.0)
-      net-sftp (>= 2.0.0)
-      net-ssh (>= 2.0.14)
-      net-ssh-gateway (>= 1.1.0)
     concurrent-ruby (1.1.9)
     crass (1.0.6)
     erubi (1.10.0)
@@ -100,8 +94,8 @@ GEM
     faraday (0.15.4)
       multipart-post (>= 1.2, < 3)
     ffi (1.9.25)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
+    globalid (1.0.0)
+      activesupport (>= 5.0)
     googleauth (0.9.0)
       faraday (~> 0.12)
       jwt (>= 1.4, < 3.0)
@@ -109,7 +103,6 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    highline (2.0.1)
     httpclient (2.8.3)
     i18n (0.9.5)
       concurrent-ruby (~> 1.0)
@@ -130,32 +123,25 @@ GEM
       railties (>= 4)
       request_store (~> 1.0)
     logstash-event (1.2.02)
-    loofah (2.10.0)
+    loofah (2.14.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    marcel (1.0.1)
+    marcel (1.0.2)
     memoist (0.16.2)
     metaclass (0.0.4)
     method_source (1.0.0)
-    mini_mime (1.1.0)
-    mini_portile2 (2.6.1)
+    mini_mime (1.1.2)
+    mini_portile2 (2.8.0)
     minitest (5.10.3)
     mocha (1.8.0)
       metaclass (~> 0.0.1)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    net-scp (2.0.0)
-      net-ssh (>= 2.6.5, < 6.0.0)
-    net-sftp (2.1.2)
-      net-ssh (>= 2.6.5)
-    net-ssh (5.2.0)
-    net-ssh-gateway (2.0.0)
-      net-ssh (>= 4.0.0)
-    nio4r (2.5.7)
-    nokogiri (1.12.5)
-      mini_portile2 (~> 2.6.1)
+    nio4r (2.5.8)
+    nokogiri (1.13.3)
+      mini_portile2 (~> 2.8.0)
       racc (~> 1.4)
     oj (3.9.2)
     optimist (3.0.0)
@@ -170,18 +156,18 @@ GEM
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.6)
-      actioncable (= 5.2.6)
-      actionmailer (= 5.2.6)
-      actionpack (= 5.2.6)
-      actionview (= 5.2.6)
-      activejob (= 5.2.6)
-      activemodel (= 5.2.6)
-      activerecord (= 5.2.6)
-      activestorage (= 5.2.6)
-      activesupport (= 5.2.6)
+    rails (5.2.6.3)
+      actioncable (= 5.2.6.3)
+      actionmailer (= 5.2.6.3)
+      actionpack (= 5.2.6.3)
+      actionview (= 5.2.6.3)
+      activejob (= 5.2.6.3)
+      activemodel (= 5.2.6.3)
+      activerecord (= 5.2.6.3)
+      activestorage (= 5.2.6.3)
+      activesupport (= 5.2.6.3)
       bundler (>= 1.3.0)
-      railties (= 5.2.6)
+      railties (= 5.2.6.3)
       sprockets-rails (>= 2.0.0)
     rails-controller-testing (1.0.4)
       actionpack (>= 5.0.1.x)
@@ -190,18 +176,18 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
+    rails-html-sanitizer (1.4.2)
       loofah (~> 2.3)
     rails-observers (0.1.5)
       activemodel (>= 4.0)
     rails-perftest (0.0.7)
-    railties (5.2.6)
-      actionpack (= 5.2.6)
-      activesupport (= 5.2.6)
+    railties (5.2.6.3)
+      actionpack (= 5.2.6.3)
+      activesupport (= 5.2.6.3)
       method_source
       rake (>= 0.8.7)
       thor (>= 0.19.0, < 2.0)
-    rake (13.0.3)
+    rake (13.0.6)
     rb-fsevent (0.10.3)
     rb-inotify (0.9.10)
       ffi (>= 0.5.0, < 2)
@@ -212,8 +198,6 @@ GEM
       railties (>= 4.2.0, < 6.0)
     retriable (1.4.1)
     ruby-prof (0.15.9)
-    rvm-capistrano (1.5.6)
-      capistrano (~> 2.15.4)
     safe_yaml (1.0.5)
     signet (0.11.0)
       addressable (~> 2.3)
@@ -236,11 +220,11 @@ GEM
     sshkey (2.0.0)
     test-unit (3.3.1)
       power_assert
-    thor (1.1.0)
+    thor (1.2.1)
     thread_safe (0.3.6)
     tzinfo (1.2.9)
       thread_safe (~> 0.1)
-    websocket-driver (0.7.4)
+    websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
 
@@ -271,7 +255,6 @@ DEPENDENCIES
   rails-perftest
   responders (~> 2.0)
   ruby-prof (~> 0.15.0)
-  rvm-capistrano
   safe_yaml
   signet (< 0.12)
   simplecov (~> 0.7.1)
diff --git a/services/api/app/controllers/arvados/v1/schema_controller.rb b/services/api/app/controllers/arvados/v1/schema_controller.rb
index 59ac639baf..5508ac0fbd 100644
--- a/services/api/app/controllers/arvados/v1/schema_controller.rb
+++ b/services/api/app/controllers/arvados/v1/schema_controller.rb
@@ -37,7 +37,7 @@ class Arvados::V1::SchemaController < ApplicationController
         # format is YYYYMMDD, must be fixed width (needs to be lexically
         # sortable), updated manually, may be used by clients to
         # determine availability of API server features.
-        revision: "20210628",
+        revision: "20220222",
         source_version: AppVersion.hash,
         sourceVersion: AppVersion.hash, # source_version should be deprecated in the future
         packageVersion: AppVersion.package_version,
@@ -406,6 +406,20 @@ class Arvados::V1::SchemaController < ApplicationController
         end
       end
 
+      # The 'replace_files' option is implemented in lib/controller,
+      # not Rails -- we just need to add it here so discovery-aware
+      # clients know how to validate it.
+      [:create, :update].each do |action|
+        discovery[:resources]['collections'][:methods][action][:parameters]['replace_files'] = {
+          type: 'object',
+          description: 'Files and directories to initialize/replace with content from other collections.',
+          required: false,
+          location: 'query',
+          properties: {},
+          additionalProperties: {type: 'string'},
+        }
+      end
+
       discovery[:resources]['configs'] = {
         methods: {
           get: {
@@ -427,6 +441,27 @@ class Arvados::V1::SchemaController < ApplicationController
         }
       }
 
+      discovery[:resources]['vocabularies'] = {
+        methods: {
+          get: {
+            id: "arvados.vocabularies.get",
+            path: "vocabulary",
+            httpMethod: "GET",
+            description: "Get vocabulary definition",
+            parameters: {
+            },
+            parameterOrder: [
+            ],
+            response: {
+            },
+            scopes: [
+              "https://api.arvados.org/auth/arvados",
+              "https://api.arvados.org/auth/arvados.readonly"
+            ]
+          },
+        }
+      }
+
       discovery[:resources]['sys'] = {
         methods: {
           get: {
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index c74c1ce5bf..993a49e5b7 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -35,7 +35,12 @@ class ApiClientAuthorization < ArvadosModel
   UNLOGGED_CHANGES = ['last_used_at', 'last_used_by_ip_address', 'updated_at']
 
   def assign_random_api_token
-    self.api_token ||= rand(2**256).to_s(36)
+    begin
+      self.api_token ||= rand(2**256).to_s(36)
+    rescue ActiveModel::MissingAttributeError
+      # Ignore the case where self.api_token doesn't exist, which happens when
+      # the select=[...] is used.
+    end
   end
 
   def owner_uuid
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index 44e6ca7578..bbb2378f5c 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -21,6 +21,7 @@ class User < ArvadosModel
             uniqueness: true,
             allow_nil: true)
   validate :must_unsetup_to_deactivate
+  validate :identity_url_nil_if_empty
   before_update :prevent_privilege_escalation
   before_update :prevent_inactive_admin
   before_update :verify_repositories_empty, :if => Proc.new {
@@ -828,4 +829,10 @@ SELECT target_uuid, perm_level
       repo.save!
     end
   end
+
+  def identity_url_nil_if_empty
+    if identity_url == ""
+      self.identity_url = nil
+    end
+  end
 end
diff --git a/services/api/lib/tasks/delete_old_container_logs.rake b/services/api/lib/tasks/delete_old_container_logs.rake
index 2146d9bc37..7a0ab3826a 100644
--- a/services/api/lib/tasks/delete_old_container_logs.rake
+++ b/services/api/lib/tasks/delete_old_container_logs.rake
@@ -11,7 +11,7 @@ namespace :db do
   desc "Remove old container log entries from the logs table"
 
   task delete_old_container_logs: :environment do
-    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND clock_timestamp() - containers.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
+    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN containers ON logs.object_uuid = containers.uuid WHERE event_type IN ('stdout', 'stderr', 'arv-mount', 'crunch-run', 'crunchstat') AND containers.log IS NOT NULL AND now() - containers.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
 
     ActiveRecord::Base.connection.execute(delete_sql)
   end
diff --git a/services/api/lib/tasks/delete_old_job_logs.rake b/services/api/lib/tasks/delete_old_job_logs.rake
deleted file mode 100644
index a1ae2226a0..0000000000
--- a/services/api/lib/tasks/delete_old_job_logs.rake
+++ /dev/null
@@ -1,16 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-# This task finds jobs that have been finished for at least as long as
-# the duration specified in the `clean_job_log_rows_after`
-# configuration setting, and deletes their stderr logs from the logs table.
-
-namespace :db do
-  desc "Remove old job stderr entries from the logs table"
-  task delete_old_job_logs: :environment do
-    delete_sql = "DELETE FROM logs WHERE id in (SELECT logs.id FROM logs JOIN jobs ON logs.object_uuid = jobs.uuid WHERE event_type = 'stderr' AND jobs.log IS NOT NULL AND clock_timestamp() - jobs.finished_at > interval '#{Rails.configuration.Containers.Logging.MaxAge.to_i} seconds')"
-
-    ActiveRecord::Base.connection.execute(delete_sql)
-  end
-end
diff --git a/services/api/lib/tasks/symbols.rake b/services/api/lib/tasks/symbols.rake
deleted file mode 100644
index dc9ed461dd..0000000000
--- a/services/api/lib/tasks/symbols.rake
+++ /dev/null
@@ -1,114 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-# This is needed instead of just including CurrentApiClient so that its
-# methods don't get imported as Object's class methods; this is a problem because
-# the methods would be imported only on test environment. See #15716 for more info.
-class CurrentApiClientHelper
-  extend CurrentApiClient
-end
-
-def has_symbols? x
-  if x.is_a? Hash
-    x.each do |k,v|
-      return true if has_symbols?(k) or has_symbols?(v)
-    end
-  elsif x.is_a? Array
-    x.each do |k|
-      return true if has_symbols?(k)
-    end
-  elsif x.is_a? Symbol
-    return true
-  elsif x.is_a? String
-    return true if x.start_with?(':') && !x.start_with?('::')
-  end
-  false
-end
-
-def check_for_serialized_symbols rec
-  jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
-  (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
-    if has_symbols? rec.attributes[colname]
-      st = recursive_stringify rec.attributes[colname]
-      puts "Found value potentially containing Ruby symbols in #{colname} attribute of #{rec.uuid}, current value is\n#{rec.attributes[colname].to_s[0..1024]}\nrake symbols:stringify will update it to:\n#{st.to_s[0..1024]}\n\n"
-    end
-  end
-end
-
-def recursive_stringify x
-  if x.is_a? Hash
-    Hash[x.collect do |k,v|
-           [recursive_stringify(k), recursive_stringify(v)]
-         end]
-  elsif x.is_a? Array
-    x.collect do |k|
-      recursive_stringify k
-    end
-  elsif x.is_a? Symbol
-    x.to_s
-  elsif x.is_a? String and x.start_with?(':') and !x.start_with?('::')
-    x[1..-1]
-  else
-    x
-  end
-end
-
-def stringify_serialized_symbols rec
-  # ensure_serialized_attribute_type should prevent symbols from
-  # getting into the database in the first place. If someone managed
-  # to get them into the database (perhaps using an older version)
-  # we'll convert symbols to strings when loading from the
-  # database. (Otherwise, loading and saving an object with existing
-  # symbols in a serialized field will crash.)
-  jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
-  (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
-    if has_symbols? rec.attributes[colname]
-      begin
-        st = recursive_stringify rec.attributes[colname]
-        puts "Updating #{colname} attribute of #{rec.uuid} from\n#{rec.attributes[colname].to_s[0..1024]}\nto\n#{st.to_s[0..1024]}\n\n"
-        rec.write_attribute(colname, st)
-        rec.save!
-      rescue => e
-        puts "Failed to update #{rec.uuid}: #{e}"
-      end
-    end
-  end
-end
-
-namespace :symbols do
-  desc 'Warn about serialized values starting with ":" that may be symbols'
-  task check: :environment do
-    [ApiClientAuthorization, ApiClient,
-     AuthorizedKey, Collection,
-     Container, ContainerRequest, Group,
-     Human, Job, JobTask, KeepDisk, KeepService, Link,
-     Node, PipelineInstance, PipelineTemplate,
-     Repository, Specimen, Trait, User, VirtualMachine,
-     Workflow].each do |klass|
-      CurrentApiClientHelper.act_as_system_user do
-        klass.all.each do |c|
-          check_for_serialized_symbols c
-        end
-      end
-    end
-  end
-
-  task stringify: :environment do
-    [ApiClientAuthorization, ApiClient,
-     AuthorizedKey, Collection,
-     Container, ContainerRequest, Group,
-     Human, Job, JobTask, KeepDisk, KeepService, Link,
-     Node, PipelineInstance, PipelineTemplate,
-     Repository, Specimen, Trait, User, VirtualMachine,
-     Workflow].each do |klass|
-      CurrentApiClientHelper.act_as_system_user do
-        klass.all.each do |c|
-          stringify_serialized_symbols c
-        end
-      end
-    end
-  end
-end
diff --git a/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb b/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb
index bf407afcd7..9c70f6f417 100644
--- a/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/api_client_authorizations_controller_test.rb
@@ -203,4 +203,20 @@ class Arvados::V1::ApiClientAuthorizationsControllerTest < ActionController::Tes
     get :current
     assert_response 401
   end
+
+  # Tests regression #18801
+  test "select param is respected in 'show' response" do
+    authorize_with :active
+    get :show, params: {
+          id: api_client_authorizations(:active).uuid,
+          select: ["uuid"],
+        }
+    assert_response :success
+    assert_raises ActiveModel::MissingAttributeError do
+      assigns(:object).api_token
+    end
+    assert_nil json_response["expires_at"]
+    assert_nil json_response["api_token"]
+    assert_equal api_client_authorizations(:active).uuid, json_response["uuid"]
+  end
 end
diff --git a/services/api/test/tasks/delete_old_job_logs_test.rb b/services/api/test/tasks/delete_old_job_logs_test.rb
deleted file mode 100644
index 00660431c3..0000000000
--- a/services/api/test/tasks/delete_old_job_logs_test.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'test_helper'
-require 'rake'
-
-Rake.application.rake_require "tasks/delete_old_job_logs"
-Rake::Task.define_task(:environment)
-
-class DeleteOldJobLogsTaskTest < ActiveSupport::TestCase
-  TASK_NAME = "db:delete_old_job_logs"
-
-  def log_uuids(*fixture_names)
-    fixture_names.map { |name| logs(name).uuid }
-  end
-
-  def run_with_expiry(clean_after)
-    Rails.configuration.Containers.Logging.MaxAge = clean_after
-    Rake::Task[TASK_NAME].reenable
-    Rake.application.invoke_task TASK_NAME
-  end
-
-  def job_stderr_logs
-    Log.where("object_uuid LIKE :pattern AND event_type = :etype",
-              pattern: "_____-8i9sb-_______________",
-              etype: "stderr")
-  end
-
-  def check_existence(test_method, fixture_uuids)
-    uuids_now = job_stderr_logs.map(&:uuid)
-    fixture_uuids.each do |expect_uuid|
-      send(test_method, uuids_now, expect_uuid)
-    end
-  end
-
-  test "delete all logs" do
-    uuids_to_keep = log_uuids(:crunchstat_for_running_job)
-    uuids_to_clean = log_uuids(:crunchstat_for_previous_job,
-                               :crunchstat_for_ancient_job)
-    run_with_expiry(1)
-    check_existence(:assert_includes, uuids_to_keep)
-    check_existence(:refute_includes, uuids_to_clean)
-  end
-
-  test "delete only old logs" do
-    uuids_to_keep = log_uuids(:crunchstat_for_running_job,
-                              :crunchstat_for_previous_job)
-    uuids_to_clean = log_uuids(:crunchstat_for_ancient_job)
-    run_with_expiry(360.days)
-    check_existence(:assert_includes, uuids_to_keep)
-    check_existence(:refute_includes, uuids_to_clean)
-  end
-end
diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb
index 7368d89374..9a0e1dbf9c 100644
--- a/services/api/test/unit/user_test.rb
+++ b/services/api/test/unit/user_test.rb
@@ -797,4 +797,12 @@ class UserTest < ActiveSupport::TestCase
     assert user.save
   end
 
+  test "empty identity_url saves as null" do
+    set_user_from_auth :admin
+    user = users(:active)
+    assert user.update_attributes(identity_url: '')
+    user.reload
+    assert_nil user.identity_url
+  end
+
 end
diff --git a/tools/arvbash/arvbash.sh b/tools/arvbash/arvbash.sh
index ecad0888df..1d4fbade8b 100755
--- a/tools/arvbash/arvbash.sh
+++ b/tools/arvbash/arvbash.sh
@@ -15,10 +15,10 @@ Syntax:
 arvswitch 
   Set ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment based on
   $HOME/.config/arvados/.conf
-  With no arguments, list available Arvados configurations.
+  With no arguments, print current API host and available Arvados configurations.
 
 arvsave 
-  Save values of ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment to
+  Save current values of ARVADOS_API_HOST and ARVADOS_API_TOKEN in the current environment to
   $HOME/.config/arvados/.conf
 
 arvrm 
@@ -26,12 +26,12 @@ arvrm 
 
 arvboxswitch 
   Set ARVBOX_CONTAINER to 
-  With no arguments, list available arvboxes.
+  With no arguments, print current arvbox and available arvboxes.
 
-arvopen:
+arvopen 
   Open an Arvados uuid in web browser (http://arvadosapi.com)
 
-arvissue
+arvissue 
   Open an Arvados ticket in web browser (http://dev.arvados.org)
 
 EOF
@@ -61,7 +61,8 @@ arvswitch() {
         fi
     else
         echo "Switch Arvados environment conf"
-        echo "Usage: arvswitch name"
+	echo "Current host: ${ARVADOS_API_HOST}"
+        echo "Usage: arvswitch "
         echo "Available confs:" $((cd $HOME/.config/arvados && ls --indicator-style=none *.conf) | rev | cut -c6- | rev)
     fi
 }
@@ -73,7 +74,7 @@ arvsave() {
         env | grep ARVADOS_ > $HOME/.config/arvados/$1.conf
     else
         echo "Save current Arvados environment variables to conf file"
-        echo "Usage: arvsave name"
+        echo "Usage: arvsave "
     fi
 }
 
@@ -86,25 +87,25 @@ arvrm() {
         fi
     else
         echo "Delete Arvados environment conf"
-        echo "Usage: arvrm name"
+        echo "Usage: arvrm "
     fi
 }
 
 arvboxswitch() {
     if [[ -n "$1" ]] ; then
+        export ARVBOX_CONTAINER=$1
         if [[ -d $HOME/.arvbox/$1 ]] ; then
-            export ARVBOX_CONTAINER=$1
             echo "Arvbox switched to $1"
         else
-            echo "$1 unknown"
+            echo "Warning: $1 doesn't exist, will be created."
         fi
     else
         if test -z "$ARVBOX_CONTAINER" ; then
             ARVBOX_CONTAINER=arvbox
         fi
         echo "Switch Arvbox environment conf"
-        echo "Usage: arvboxswitch name"
         echo "Your current container is: $ARVBOX_CONTAINER"
+        echo "Usage: arvboxswitch "
         echo "Available confs:" $(cd $HOME/.arvbox && ls --indicator-style=none)
     fi
 }
@@ -114,7 +115,7 @@ arvopen() {
         xdg-open https://arvadosapi.com/$1
     else
         echo "Open Arvados uuid in browser"
-        echo "Usage: arvopen uuid"
+        echo "Usage: arvopen "
     fi
 }
 
@@ -123,6 +124,6 @@ arvissue() {
         xdg-open https://dev.arvados.org/issues/$1
     else
         echo "Open Arvados issue in browser"
-        echo "Usage: arvissue uuid"
+        echo "Usage: arvissue "
     fi
 }
diff --git a/tools/arvbox/lib/arvbox/docker/createusers.sh b/tools/arvbox/lib/arvbox/docker/createusers.sh
index 9c81a66ced..4cafd8c09c 100755
--- a/tools/arvbox/lib/arvbox/docker/createusers.sh
+++ b/tools/arvbox/lib/arvbox/docker/createusers.sh
@@ -34,20 +34,12 @@ if ! grep "^arvbox:" /etc/passwd >/dev/null 2>/dev/null ; then
         chown arvbox:arvbox -R /usr/local $ARVADOS_CONTAINER_PATH \
               /var/lib/passenger /var/lib/postgresql \
               /var/lib/nginx /var/log/nginx /etc/ssl/private \
-              /var/lib/gopath /var/lib/pip /var/lib/npm \
-              /var/lib/arvados
+              /var/lib/gopath /var/lib/pip /var/lib/npm
     fi
 
     mkdir -p /tmp/crunch0 /tmp/crunch1
     chown crunch:crunch -R /tmp/crunch0 /tmp/crunch1
 
-    # singularity needs to be owned by root and suid
-    chown root /var/lib/arvados/bin/singularity \
-	  /var/lib/arvados/etc/singularity/singularity.conf \
-	  /var/lib/arvados/etc/singularity/capability.json \
-	  /var/lib/arvados/etc/singularity/ecl.toml
-    chmod u+s /var/lib/arvados/bin/singularity
-
     echo "arvbox    ALL=(crunch) NOPASSWD: ALL" >> /etc/sudoers
 
     cat < /etc/profile.d/paths.sh
diff --git a/tools/arvbox/lib/arvbox/docker/go-setup.sh b/tools/arvbox/lib/arvbox/docker/go-setup.sh
index 5bdc5207a3..03eac65cec 100644
--- a/tools/arvbox/lib/arvbox/docker/go-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/go-setup.sh
@@ -13,6 +13,7 @@ fi
 
 if [[ ! -f /usr/local/bin/arvados-server ]]; then
   $RUNSU flock /var/lib/gopath/gopath.lock go mod download
+  $RUNSU flock /var/lib/gopath/gopath.lock go mod vendor
   $RUNSU flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
   $RUNSU flock /var/lib/gopath/gopath.lock install $GOPATH/bin/arvados-server /usr/local/bin
 fi
diff --git a/tools/compute-images/arvados-images-aws.json b/tools/compute-images/arvados-images-aws.json
index 131aa8a878..94cb24adf9 100644
--- a/tools/compute-images/arvados-images-aws.json
+++ b/tools/compute-images/arvados-images-aws.json
@@ -25,7 +25,7 @@
     "region": "{{user `aws_default_region`}}",
     "ena_support": "true",
     "source_ami": "{{user `aws_source_ami`}}",
-    "instance_type": "m4.large",
+    "instance_type": "m5.large",
     "vpc_id": "{{user `vpc_id`}}",
     "subnet_id": "{{user `subnet_id`}}",
     "associate_public_ip_address": "{{user `associate_public_ip_address`}}",
@@ -34,7 +34,7 @@
     "launch_block_device_mappings": [{
       "device_name": "/dev/xvda",
       "volume_size": 20,
-      "volume_type": "gp2",
+      "volume_type": "gp3",
       "delete_on_termination": true
     }],
     "ami_block_device_mappings": [
diff --git a/tools/compute-images/scripts/base.sh b/tools/compute-images/scripts/base.sh
index 90b845f1ac..260c5d47ee 100644
--- a/tools/compute-images/scripts/base.sh
+++ b/tools/compute-images/scripts/base.sh
@@ -15,6 +15,9 @@ wait_for_apt_locks() {
   done
 }
 
+# $DIST should not have a dot if there is one in /etc/os-release (e.g. 18.04)
+DIST=$(. /etc/os-release; echo $ID$VERSION_ID | tr -d '.')
+
 # Run apt-get update
 $SUDO DEBIAN_FRONTEND=noninteractive apt-get --yes update
 
@@ -36,6 +39,11 @@ fi
 TMP_LSB=`/usr/bin/lsb_release -c -s`
 LSB_RELEASE_CODENAME=${TMP_LSB//[$'\t\r\n ']}
 
+SET_RESOLVER=
+if [ -n "$RESOLVER" ]; then
+  SET_RESOLVER="--dns ${RESOLVER}"
+fi
+
 # Add the arvados apt repository
 echo "# apt.arvados.org" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
 echo "deb http://apt.arvados.org/$LSB_RELEASE_CODENAME $LSB_RELEASE_CODENAME${REPOSUFFIX} main" |$SUDO tee --append /etc/apt/sources.list.d/apt.arvados.org.list
@@ -66,8 +74,45 @@ wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes ins
 # Install the Arvados packages we need
 wait_for_apt_locks && $SUDO DEBIAN_FRONTEND=noninteractive apt-get -qq --yes install \
   python3-arvados-fuse \
-  arvados-docker-cleaner \
-  docker.io
+  arvados-docker-cleaner
+
+# We want Docker 20.10 or later so that we support glibc 2.33 and up in the container, cf.
+# https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=1005906
+dockerversion=5:20.10.13~3-0
+if [[ "$DIST" =~ ^debian ]]; then
+  family="debian"
+  if [ "$DIST" == "debian10" ]; then
+    distro="buster"
+  elif [ "$DIST" == "debian11" ]; then
+    distro="bullseye"
+  fi
+elif [[ "$DIST" =~ ^ubuntu ]]; then
+  family="ubuntu"
+  if [ "$DIST" == "ubuntu1804" ]; then
+    distro="bionic"
+  elif [ "$DIST" == "ubuntu2004" ]; then
+    distro="focal"
+  fi
+else
+  echo "Unsupported distribution $DIST"
+  exit 1
+fi
+curl -fsSL https://download.docker.com/linux/$family/gpg | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+echo deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/$family/ $distro stable | \
+    $SUDO tee /etc/apt/sources.list.d/docker.list
+$SUDO apt-get update
+$SUDO apt-get -yq --no-install-recommends install docker-ce=${dockerversion}~${family}-${distro}
+
+# Set a higher ulimit and the resolver (if set) for docker
+$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
+  /lib/systemd/system/docker.service \
+  > /etc/systemd/system/docker.service
+
+$SUDO systemctl daemon-reload
+
+# docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
+# and the BootProbeCommand might be "docker ps -q"
+$SUDO systemctl disable docker
 
 # Get Go and build singularity
 goversion=1.17.1
@@ -109,21 +154,6 @@ $SUDO echo -e "{\n  \"Quota\": \"10G\",\n  \"RemoveStoppedContainers\": \"always
 $SUDO sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="cgroup_enable=memory swapaccount=1"/g' /etc/default/grub
 $SUDO update-grub
 
-# Set a higher ulimit and the resolver (if set) for docker
-if [ "x$RESOLVER" != "x" ]; then
-  SET_RESOLVER="--dns ${RESOLVER}"
-fi
-
-$SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
-  /lib/systemd/system/docker.service \
-  > /etc/systemd/system/docker.service
-
-$SUDO systemctl daemon-reload
-
-# docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
-# and the BootProbeCommand might be "docker ps -q"
-$SUDO systemctl disable docker
-
 # Make sure user_allow_other is set in fuse.conf
 $SUDO sed -i 's/#user_allow_other/user_allow_other/g' /etc/fuse.conf
 
@@ -145,7 +175,10 @@ if [ "x$RESOLVER" != "x" ]; then
   $SUDO sed -i "s/#prepend domain-name-servers 127.0.0.1;/prepend domain-name-servers ${RESOLVER};/" /etc/dhcp/dhclient.conf
 fi
 
-if [ "$AWS_EBS_AUTOSCALE" != "1" ]; then
+# AWS_EBS_AUTOSCALE is not always set, work around unset variable check
+EBS_AUTOSCALE=${AWS_EBS_AUTOSCALE:-}
+
+if [ "$EBS_AUTOSCALE" != "1" ]; then
   # Set up the cloud-init script that will ensure encrypted disks
   $SUDO mv /tmp/usr-local-bin-ensure-encrypted-partitions.sh /usr/local/bin/ensure-encrypted-partitions.sh
 else
@@ -173,8 +206,6 @@ $SUDO mv /tmp/etc-cloud-cloud.cfg.d-07_compute_arvados_dispatch_cloud.cfg /etc/c
 $SUDO chown root:root /etc/cloud/cloud.cfg.d/07_compute_arvados_dispatch_cloud.cfg
 
 if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
-  # $DIST should not have a dot if there is one in /etc/os-release (e.g. 18.04)
-  DIST=$(. /etc/os-release; echo $ID$VERSION_ID | tr -d '.')
   # We need a kernel and matching headers
   if [[ "$DIST" =~ ^debian ]]; then
     $SUDO apt-get -y install linux-image-cloud-amd64 linux-headers-cloud-amd64
@@ -188,7 +219,8 @@ if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
   $SUDO apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/7fa2af80.pub
   $SUDO apt-get -y install software-properties-common
   $SUDO add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/$DIST/x86_64/ /"
-  $SUDO add-apt-repository contrib
+  # Ubuntu 18.04's add-apt-repository does not understand 'contrib'
+  $SUDO add-apt-repository contrib || true
   $SUDO apt-get update
   $SUDO apt-get -y install cuda
 
@@ -210,24 +242,6 @@ if [ "$NVIDIA_GPU_SUPPORT" == "1" ]; then
       $SUDO tee /etc/apt/sources.list.d/libnvidia-container.list
   fi
 
-  if [ "$DIST" == "debian10" ]; then
-    # Debian 10 comes with Docker 18.xx, we need 19.03 or later
-    curl -fsSL https://download.docker.com/linux/debian/gpg | $SUDO gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
-    echo deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/debian/ buster stable | \
-      $SUDO tee /etc/apt/sources.list.d/docker.list
-    $SUDO apt-get update
-    $SUDO apt-get -yq --no-install-recommends install docker-ce=5:19.03.15~3-0~debian-buster
-
-    $SUDO sed "s/ExecStart=\(.*\)/ExecStart=\1 --default-ulimit nofile=10000:10000 ${SET_RESOLVER}/g" \
-      /lib/systemd/system/docker.service \
-      > /etc/systemd/system/docker.service
-
-    $SUDO systemctl daemon-reload
-
-    # docker should not start on boot: we restart it inside /usr/local/bin/ensure-encrypted-partitions.sh,
-    # and the BootProbeCommand might be "docker ps -q"
-    $SUDO systemctl disable docker
-  fi
   $SUDO apt-get update
   $SUDO apt-get -y install libnvidia-container1 libnvidia-container-tools nvidia-container-toolkit
   # This service fails to start when the image is booted without Nvidia GPUs present, which makes
diff --git a/tools/crunchstat-summary/setup.py b/tools/crunchstat-summary/setup.py
index 8507990f5a..a881390e47 100755
--- a/tools/crunchstat-summary/setup.py
+++ b/tools/crunchstat-summary/setup.py
@@ -15,6 +15,11 @@ README = os.path.join(SETUP_DIR, 'README.rst')
 
 import arvados_version
 version = arvados_version.get_version(SETUP_DIR, "crunchstat_summary")
+if os.environ.get('ARVADOS_BUILDING_VERSION', False):
+    pysdk_dep = "=={}".format(version)
+else:
+    # On dev releases, arvados-python-client may have a different timestamp
+    pysdk_dep = "<={}".format(version)
 
 short_tests_only = False
 if '--short-tests-only' in sys.argv:
@@ -38,7 +43,7 @@ setup(name='crunchstat_summary',
           ('share/doc/crunchstat_summary', ['agpl-3.0.txt']),
       ],
       install_requires=[
-          'arvados-python-client',
+          'arvados-python-client{}'.format(pysdk_dep),
       ],
       test_suite='tests',
       tests_require=['pbr<1.7.0', 'mock>=1.0'],
diff --git a/tools/salt-install/Vagrantfile b/tools/salt-install/Vagrantfile
index f5759c482d..1573b6862b 100644
--- a/tools/salt-install/Vagrantfile
+++ b/tools/salt-install/Vagrantfile
@@ -37,10 +37,13 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
                                     s#domain_fixme_or_this_wont_work#local#g;
                                     s#CONTROLLER_EXT_SSL_PORT=443#CONTROLLER_EXT_SSL_PORT=8443#g;
                                     s#RELEASE=\"production\"#RELEASE=\"development\"#g;
-                                    s/# VERSION=.*$/VERSION=\"latest\"/g;
-                                    s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g' \
+                                    s/# VERSION=.*$/VERSION=\"latest\"/g;' \
                                     /vagrant/local.params.example.single_host_multiple_hostnames > /tmp/local.params.single_host_multiple_hostnames"
+                                    # s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;' \
 
+     arv.vm.provision "shell",
+                      inline: "cp -vr /tmp/local.params.single_host_multiple_hostnames /tmp/local.params.single_host_multiple_hostnames.falla;
+                               cp -vr /vagrant/centos7-local.params.single_host_single_hostname-f258b604f831bb3bd7fab506c670b975ae8e4118 /tmp/local.params.single_host_multiple_hostnames"
      arv.vm.provision "shell",
                       path: "provision.sh",
                       args: [
@@ -54,7 +57,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
    # A single_host single_hostname example
    config.vm.define "arvados-sh-sn" do |arv|
-     arv.vm.box = "bento/debian-10"
+     #arv.vm.box = "bento/centos-7"
+     arv.vm.box = "bento/ubuntu-20.04"
      arv.vm.hostname = "zeppo"
      # CPU/RAM
      config.vm.provider :virtualbox do |v|
@@ -80,15 +84,19 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
      arv.vm.provision "shell",
                       inline: "cp -vr /vagrant/config_examples/single_host/single_hostname /home/vagrant/local_config_dir;
                                cp -vr /vagrant/tests /home/vagrant/tests;
-                               sed 's#HOSTNAME_EXT=\"\"#HOSTNAME_EXT=\"zeppo.local\"#g;
-                                    s#cluster_fixme_or_this_wont_work#zeppo#g;
-                                    s/#\ BRANCH=\"main\"/\ BRANCH=\"main\"/g;
-                                    s#domain_fixme_or_this_wont_work#local#g;' \
+                               sed 's#cluster_fixme_or_this_wont_work#cnts7#g;
+                                    s#domain_fixme_or_this_wont_work#local#g;
+                                    s#HOSTNAME_EXT=\"hostname_ext_fixme_or_this_wont_work\"#HOSTNAME_EXT=\"cnts7.local\"#g;
+                                    s#IP_INT=\"ip_int_fixme_or_this_wont_work\"#IP_INT=\"127.0.0.1\"#g;
+                                    s#RELEASE=\"production\"#RELEASE=\"development\"#g;
+                                    s/# BRANCH=\"main\"/BRANCH=\"main\"/g;
+                                    s/# VERSION=.*$/VERSION=\"latest\"/g' \
                                     /vagrant/local.params.example.single_host_single_hostname > /tmp/local.params.single_host_single_hostname"
+
      arv.vm.provision "shell",
                       path: "provision.sh",
                       args: [
-                        # "--debug",
+                        "--debug",
                         "--config /tmp/local.params.single_host_single_hostname",
                         "--test",
                         "--vagrant"
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
index f27aa40ac4..8c14c56ed3 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
@@ -157,6 +157,9 @@ arvados:
       DispatchCloud:
         InternalURLs:
           'http://__CONTROLLER_INT_IP__:9006': {}
+      Keepbalance:
+        InternalURLs:
+          'http://localhost:9005': {}
       Keepproxy:
         ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__KEEP_EXT_SSL_PORT__'
         InternalURLs:
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
index 28cc748dac..fbd42bd7a3 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
@@ -10,12 +10,12 @@
                           if grains.osfinger in ('CentOS Linux-7',) else
                         '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
 {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
-                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
                          '/usr/bin/ruby' %}
 
 ### NGINX
 nginx:
-  install_from_phusionpassenger: true
+  __NGINX_INSTALL_SOURCE__: true
   lookup:
     passenger_package: {{ passenger_pkg }}
   ### PASSENGER
@@ -25,11 +25,15 @@ nginx:
   ### SERVER
   server:
     config:
+      # As we now differentiate where passenger is required or not, we need to
+      # load this module conditionally, so we add this conditional just to use
+      # the same pillar file
+      {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
       # This is required to get the passenger module loaded
       # In Debian it can be done with this
       # include: 'modules-enabled/*.conf'
       load_module: {{ passenger_mod }}
-
+      {% endif %}
       worker_processes: 4
 
   ### SNIPPETS
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
index a0da9a1c05..e06ddd041c 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/postgresql.sls
@@ -6,7 +6,7 @@
 ### POSTGRESQL
 postgres:
   use_upstream_repo: true
-  version: '11'
+  version: '12'
   postgresconf: |-
     listen_addresses = '*'  # listen on all interfaces
   acls:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 81d324fcbd..2579c5ffb0 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -142,7 +142,7 @@ arvados:
           'http://__CLUSTER__.__DOMAIN__:9006': {}
       Keepbalance:
         InternalURLs:
-          'http://__CLUSTER__.__DOMAIN__:9005': {}
+          'http://localhost:9005': {}
       Keepproxy:
         ExternalURL: 'https://keep.__CLUSTER__.__DOMAIN__:__CONTROLLER_EXT_SSL_PORT__'
         InternalURLs:
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
index 4ad14d33ff..dbf21c2651 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/nginx_passenger.sls
@@ -10,12 +10,12 @@
                           if grains.osfinger in ('CentOS Linux-7',) else
                         '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
 {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
-                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
                          '/usr/bin/ruby' %}
 
 ### NGINX
 nginx:
-  install_from_phusionpassenger: true
+  __NGINX_INSTALL_SOURCE__: true
   lookup:
     passenger_package: {{ passenger_pkg }}
   ### PASSENGER
@@ -25,11 +25,15 @@ nginx:
   ### SERVER
   server:
     config:
+      # As we now differentiate where passenger is required or not, we need to
+      # load this module conditionally, so we add this conditional just to use
+      # the same pillar file
+      {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
       # This is required to get the passenger module loaded
       # In Debian it can be done with this
       # include: 'modules-enabled/*.conf'
       load_module: {{ passenger_mod }}
-
+      {% endif %}
       worker_processes: 4
 
   ### SNIPPETS
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/states/dns.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/dns.sls
new file mode 100644
index 0000000000..f298e8f66d
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/states/dns.sls
@@ -0,0 +1,8 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+dns:
+  pkg.installed:
+    - pkgs:
+      - dnsmasq
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
index 78a5a938f3..8e4d66caf5 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+# vim: ft=yaml
 ---
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
@@ -67,7 +69,15 @@ arvados:
       host: 127.0.0.1
       password: "__DATABASE_PASSWORD__"
       user: __CLUSTER___arvados
-      encoding: en_US.utf8
+      extra_conn_params:
+        client_encoding: UTF8
+      # Centos7 does not enable SSL by default, so we disable
+      # it here just for testing of the formula purposes only.
+      # You should not do this in production, and should
+      # configure Postgres certificates correctly
+      {%- if grains.os_family in ('RedHat',) %}
+        sslmode: disable
+      {%- endif %}
 
     tls:
       # certificate: ''
@@ -75,12 +85,18 @@ arvados:
       # When using arvados-snakeoil certs set insecure: true
       insecure: true
 
+    resources:
+      virtual_machines:
+        shell:
+          name: webshell
+          backend: 127.0.1.1
+          port: 4200
+
     ### TOKENS
     tokens:
       system_root: __SYSTEM_ROOT_TOKEN__
       management: __MANAGEMENT_TOKEN__
       anonymous_user: __ANONYMOUS_USER_TOKEN__
-      rails_secret: YDLxHf4GqqmLXYAMgndrAmFEdqgC0sBqX7TEjMN2rw9D6EVwgx
 
     ### KEYS
     secrets:
@@ -102,7 +118,7 @@ arvados:
       # -nyw5e-
       __CLUSTER__-nyw5e-000000000000000:
         AccessViaHosts:
-          'http://__HOSTNAME_INT__:25107':
+          'http://__IP_INT__:25107':
             ReadOnly: false
         Replication: 2
         Driver: Directory
@@ -119,21 +135,24 @@ arvados:
       Controller:
         ExternalURL: 'https://__HOSTNAME_EXT__:__CONTROLLER_EXT_SSL_PORT__'
         InternalURLs:
-          'http://__HOSTNAME_INT__:8003': {}
+          'http://__IP_INT__:8003': {}
+      Keepbalance:
+        InternalURLs:
+          'http://__IP_INT__:9005': {}
       Keepproxy:
         ExternalURL: 'https://__HOSTNAME_EXT__:__KEEP_EXT_SSL_PORT__'
         InternalURLs:
-          'http://__HOSTNAME_INT__:25100': {}
+          'http://__IP_INT__:25100': {}
       Keepstore:
         InternalURLs:
-          'http://__HOSTNAME_INT__:25107': {}
+          'http://__IP_INT__:25107': {}
       RailsAPI:
         InternalURLs:
-          'http://__HOSTNAME_INT__:8004': {}
+          'http://__IP_INT__:8004': {}
       WebDAV:
         ExternalURL: 'https://__HOSTNAME_EXT__:__KEEPWEB_EXT_SSL_PORT__'
         InternalURLs:
-          'http://__HOSTNAME_INT__:9003': {}
+          'http://__IP_INT__:9003': {}
       WebDAVDownload:
         ExternalURL: 'https://__HOSTNAME_EXT__:__KEEPWEB_EXT_SSL_PORT__'
       WebShell:
@@ -141,7 +160,7 @@ arvados:
       Websocket:
         ExternalURL: 'wss://__HOSTNAME_EXT__:__WEBSOCKET_EXT_SSL_PORT__/websocket'
         InternalURLs:
-          'http://__HOSTNAME_INT__:8005': {}
+          'http://__IP_INT__:8005': {}
       Workbench1:
         ExternalURL: 'https://__HOSTNAME_EXT__:__WORKBENCH1_EXT_SSL_PORT__'
       Workbench2:
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/aws_credentials.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/aws_credentials.sls
new file mode 100644
index 0000000000..35cdbf7bde
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/aws_credentials.sls
@@ -0,0 +1,9 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+aws_credentials:
+  region: __LE_AWS_REGION__
+  access_key_id: __LE_AWS_ACCESS_KEY_ID__
+  secret_access_key: __LE_AWS_SECRET_ACCESS_KEY__
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/docker.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/docker.sls
index 54d2256159..30d90153e8 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/docker.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/docker.sls
@@ -7,3 +7,4 @@ docker:
   pkg:
     docker:
       use_upstream: package
+      daemon_config: {"dns": ["__IP_INT__"]}
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/letsencrypt.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/letsencrypt.sls
new file mode 100644
index 0000000000..895c65017d
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/letsencrypt.sls
@@ -0,0 +1,24 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### LETSENCRYPT
+letsencrypt:
+  use_package: true
+  pkgs:
+    - certbot: latest
+    - python3-certbot-nginx
+  config:
+    server: https://acme-v02.api.letsencrypt.org/directory
+    email: __INITIAL_USER_EMAIL__
+    authenticator: nginx
+    agree-tos: true
+    keep-until-expiring: true
+    expand: true
+    max-log-backups: 0
+    deploy-hook: systemctl reload nginx
+
+  domainsets:
+    __HOSTNAME_EXT__:
+      - __HOSTNAME_EXT__
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_api_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_api_configuration.sls
index 18f09af503..04195ae5b9 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_api_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_api_configuration.sls
@@ -3,22 +3,28 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set group = 'nginx' %}
+{%- else %}
+  {%- set group = 'www-data' %}
+{%- endif %}
+
 ### ARVADOS
 arvados:
   config:
-    group: www-data
+    group: {{ group }}
 
 ### NGINX
 nginx:
   ### SITES
   servers:
     managed:
-      arvados_api:
+      arvados_api.conf:
         enabled: true
         overwrite: true
         config:
           - server:
-            - listen: '__HOSTNAME_INT__:8004'
+            - listen: '__IP_INT__:8004'
             - server_name: api
             - root: /var/www/arvados-api/current/public
             - index:  index.html index.htm
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls
index b7b75ab9c2..cfd1525924 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_controller_configuration.sls
@@ -14,28 +14,30 @@ nginx:
           default: 1
           '127.0.0.0/8': 0
         upstream controller_upstream:
-          - server: '__HOSTNAME_INT__:8003  fail_timeout=10s'
+          - server: '__IP_INT__:8003  fail_timeout=10s'
 
   ### SITES
   servers:
     managed:
       ### DEFAULT
-      arvados_controller_default:
+      arvados_controller_default.conf:
         enabled: true
         overwrite: true
         config:
           - server:
             - server_name: _
             - listen:
-              - 80 default_server
+              - 80
             - location /.well-known:
               - root: /var/www
             - location /:
               - return: '301 https://$host$request_uri'
 
-      arvados_controller_ssl:
+      arvados_controller_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -52,7 +54,9 @@ nginx:
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
               - proxy_set_header: 'X-External-Client $external_client'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/__CLUSTER__.__DOMAIN__.error.log
             - client_max_body_size: 128m
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepproxy_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepproxy_configuration.sls
index 81d72aac74..11f6e85695 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepproxy_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepproxy_configuration.sls
@@ -11,13 +11,27 @@ nginx:
       ### STREAMS
       http:
         upstream keepproxy_upstream:
-          - server: '__HOSTNAME_INT__:25100 fail_timeout=10s'
+          - server: '__IP_INT__:25100 fail_timeout=10s'
 
   servers:
     managed:
-      arvados_keepproxy_ssl:
+      ### DEFAULT
+      arvados_keepproxy_default.conf:
         enabled: true
         overwrite: true
+        config:
+          - server:
+            - server_name: keep.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_keepproxy_ssl.conf:
+        enabled: true
+        overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -38,6 +52,8 @@ nginx:
             - client_max_body_size: 64M
             - proxy_http_version: '1.1'
             - proxy_request_buffering: 'off'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/keepproxy.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepweb_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepweb_configuration.sls
index fcb56c9949..1082b5357d 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepweb_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_keepweb_configuration.sls
@@ -11,14 +11,16 @@ nginx:
       ### STREAMS
       http:
         upstream collections_downloads_upstream:
-          - server: '__HOSTNAME_INT__:9003 fail_timeout=10s'
+          - server: '__IP_INT__:9003 fail_timeout=10s'
 
   servers:
     managed:
       ### COLLECTIONS / DOWNLOAD
-      arvados_collections_download_ssl:
+      arvados_collections_download_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -38,6 +40,8 @@ nginx:
             - client_max_body_size: 0
             - proxy_http_version: '1.1'
             - proxy_request_buffering: 'off'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/keepweb.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/keepweb.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
index a4d3c34f26..c25720c60a 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_passenger.sls
@@ -10,12 +10,12 @@
                           if grains.osfinger in ('CentOS Linux-7',) else
                         '/usr/lib/nginx/modules/ngx_http_passenger_module.so' %}
 {%- set passenger_ruby = '/usr/local/rvm/rubies/ruby-2.7.2/bin/ruby'
-                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04',) else
+                           if grains.osfinger in ('CentOS Linux-7', 'Ubuntu-18.04', 'Debian-10') else
                          '/usr/bin/ruby' %}
 
 ### NGINX
 nginx:
-  install_from_phusionpassenger: true
+  __NGINX_INSTALL_SOURCE__: true
   lookup:
     passenger_package: {{ passenger_pkg }}
   ### PASSENGER
@@ -25,11 +25,15 @@ nginx:
   ### SERVER
   server:
     config:
+      # As we now differentiate where passenger is required or not, we need to
+      # load this module conditionally, so we add this conditional just to use
+      # the same pillar file
+      {% if "install_from_phusionpassenger" == "__NGINX_INSTALL_SOURCE__" %}
       # This is required to get the passenger module loaded
       # In Debian it can be done with this
       # include: 'modules-enabled/*.conf'
       load_module: {{ passenger_mod }}
-
+      {% endif %}
       worker_processes: 4
 
   ### SNIPPETS
@@ -69,6 +73,16 @@ nginx:
   ### SITES
   servers:
     managed:
-      # Remove default webserver
+      # Update default config to redirect to https
       default:
-        enabled: false
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: _
+            - listen:
+              - 80 default_server
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_webshell_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_webshell_configuration.sls
index 1b21aaaeb6..67013f93c2 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_webshell_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_webshell_configuration.sls
@@ -12,14 +12,16 @@ nginx:
       ### STREAMS
       http:
         upstream webshell_upstream:
-          - server: '__HOSTNAME_INT__:4200 fail_timeout=10s'
+          - server: '__IP_INT__:4200 fail_timeout=10s'
 
   ### SITES
   servers:
     managed:
-      arvados_webshell_ssl:
+      arvados_webshell_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -55,7 +57,9 @@ nginx:
                 - add_header: "'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'"
                 - add_header: "'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type'"
 
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/webshell.__CLUSTER__.__DOMAIN__.error.log
 
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_websocket_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_websocket_configuration.sls
index 7c4ff7835c..e772072174 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_websocket_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_websocket_configuration.sls
@@ -11,13 +11,15 @@ nginx:
       ### STREAMS
       http:
         upstream websocket_upstream:
-          - server: '__HOSTNAME_INT__:8005 fail_timeout=10s'
+          - server: '__IP_INT__:8005 fail_timeout=10s'
 
   servers:
     managed:
-      arvados_websocket_ssl:
+      arvados_websocket_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -39,6 +41,8 @@ nginx:
             - client_max_body_size: 64M
             - proxy_http_version: '1.1'
             - proxy_request_buffering: 'off'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/ws.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench2_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench2_configuration.sls
index 462443c1fa..d28fe80278 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench2_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench2_configuration.sls
@@ -1,21 +1,43 @@
 ---
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
-# SPDX-License-Identifier: AGPL-3.0
+# SPDX-License-Identifier: Apache-2.0
+
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set group = 'nginx' %}
+{%- else %}
+  {%- set group = 'www-data' %}
+{%- endif %}
 
 ### ARVADOS
 arvados:
   config:
-    group: www-data
+    group: {{ group }}
 
 ### NGINX
 nginx:
   ### SITES
   servers:
     managed:
-      arvados_workbench2_ssl:
+      ### DEFAULT
+      arvados_workbench2_default.conf:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench2.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_workbench2_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -29,6 +51,8 @@ nginx:
                 - return: 503
             - location /config.json:
               - return: {{ "200 '" ~ '{"API_HOST":"__HOSTNAME_EXT__:__CONTROLLER_EXT_SSL_PORT__"}' ~ "'" }}
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/workbench2.__CLUSTER__.__DOMAIN__.error.log
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench_configuration.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench_configuration.sls
index 9ed6e3b87a..59fb43e57a 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench_configuration.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/nginx_workbench_configuration.sls
@@ -3,10 +3,16 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+{%- if grains.os_family in ('RedHat',) %}
+  {%- set group = 'nginx' %}
+{%- else %}
+  {%- set group = 'www-data' %}
+{%- endif %}
+
 ### ARVADOS
 arvados:
   config:
-    group: www-data
+    group: {{ group }}
 
 ### NGINX
 nginx:
@@ -17,14 +23,30 @@ nginx:
       ### STREAMS
       http:
         upstream workbench_upstream:
-          - server: '__HOSTNAME_INT__:9000 fail_timeout=10s'
+          - server: '__IP_INT__:9000 fail_timeout=10s'
 
   ### SITES
   servers:
     managed:
-      arvados_workbench_ssl:
+      ### DEFAULT
+      arvados_workbench_default.conf:
+        enabled: true
+        overwrite: true
+        config:
+          - server:
+            - server_name: workbench.__CLUSTER__.__DOMAIN__
+            - listen:
+              - 80
+            - location /.well-known:
+              - root: /var/www
+            - location /:
+              - return: '301 https://$host$request_uri'
+
+      arvados_workbench_ssl.conf:
         enabled: true
         overwrite: true
+        requires:
+          __CERT_REQUIRES__
         config:
           - server:
             - server_name: __HOSTNAME_EXT__
@@ -40,7 +62,9 @@ nginx:
               - proxy_set_header: 'Host $http_host'
               - proxy_set_header: 'X-Real-IP $remote_addr'
               - proxy_set_header: 'X-Forwarded-For $proxy_add_x_forwarded_for'
-            - include: 'snippets/arvados-snakeoil.conf'
+            - include: snippets/ssl_hardening_default.conf
+            - ssl_certificate: __CERT_PEM__
+            - ssl_certificate_key: __CERT_KEY__
             - access_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.access.log combined
             - error_log: /var/log/nginx/workbench.__CLUSTER__.__DOMAIN__.error.log
 
@@ -49,7 +73,7 @@ nginx:
         overwrite: true
         config:
           - server:
-            - listen: '__HOSTNAME_INT__:9000'
+            - listen: '__IP_INT__:9000'
             - server_name: workbench
             - root: /var/www/arvados-workbench/current/public
             - index:  index.html index.htm
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/postgresql.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/postgresql.sls
index caafb7b2d7..a69b88cb17 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/postgresql.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/postgresql.sls
@@ -5,11 +5,32 @@
 
 ### POSTGRESQL
 postgres:
+  # Centos-7's postgres package is too old, so we need to force using upstream's
+  # This is not required in Debian's family as they already ship with PG +11
+  {%- if salt['grains.get']('os_family') == 'RedHat' %}
+  use_upstream_repo: true
+  version: '12'
+
+  pkgs_deps:
+    - libicu
+    - libxslt
+    - systemd-sysv
+
+  pkgs_extra:
+    - postgresql12-contrib
+
+  {%- else %}
   use_upstream_repo: false
   pkgs_extra:
     - postgresql-contrib
+  {%- endif %}
   postgresconf: |-
     listen_addresses = '*'  # listen on all interfaces
+    # If you want to enable communications' encryption to the DB server,
+    # uncomment these entries
+    # ssl = on
+    # ssl_cert_file = '/etc/ssl/certs/arvados-snakeoil-cert.pem'
+    # ssl_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key'
   acls:
     - ['local', 'all', 'postgres', 'peer']
     - ['local', 'all', 'all', 'peer']
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/custom_certs.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/custom_certs.sls
new file mode 100644
index 0000000000..3b2be59f36
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/custom_certs.sls
@@ -0,0 +1,33 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+{%- set orig_cert_dir = salt['pillar.get']('extra_custom_certs_dir', '/srv/salt/certs')  %}
+{%- set dest_cert_dir = '/etc/nginx/ssl' %}
+{%- set certs = salt['pillar.get']('extra_custom_certs', [])  %}
+
+{% if certs %}
+extra_custom_certs_file_directory_certs_dir:
+  file.directory:
+    - name: /etc/nginx/ssl
+    - require:
+      - pkg: nginx_install
+
+  {%- for cert in certs %}
+    {%- set cert_file = 'arvados-' ~ cert ~ '.pem' %}
+    {#- set csr_file = 'arvados-' ~ cert ~ '.csr' #}
+    {%- set key_file = 'arvados-' ~ cert ~ '.key' %}
+    {% for c in [cert_file, key_file] %}
+extra_custom_certs_file_copy_{{ c }}:
+  file.copy:
+    - name: {{ dest_cert_dir }}/{{ c }}
+    - source: {{ orig_cert_dir }}/{{ c }}
+    - force: true
+    - user: root
+    - group: root
+    - unless: cmp {{ dest_cert_dir }}/{{ c }} {{ orig_cert_dir }}/{{ c }}
+    - require:
+      - file: extra_custom_certs_file_directory_certs_dir
+    {%- endfor %}
+  {%- endfor %}
+{%- endif %}
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/dns.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/dns.sls
new file mode 100644
index 0000000000..f298e8f66d
--- /dev/null
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/dns.sls
@@ -0,0 +1,8 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+dns:
+  pkg.installed:
+    - pkgs:
+      - dnsmasq
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls
index 53a9148cc0..a688f4f8c1 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/host_entries.sls
@@ -7,12 +7,21 @@
 {%- from "arvados/map.jinja" import arvados with context %}
 {%- set tpldir = curr_tpldir %}
 
+# We need the external hostname to resolve to the internal IP for docker. We
+# tell docker to resolve via the local dnsmasq, which reads from /etc/hosts by
+# default.
+arvados_local_access_to_hostname_ext:
+  host.present:
+    - ip: __IP_INT__
+    - names:
+      - __HOSTNAME_EXT__
+
 arvados_test_salt_states_examples_single_host_etc_hosts_host_present:
   host.present:
     - ip: 127.0.1.1
     - names:
       - {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
-      # FIXME! This just works for our testings.
+      # FIXME! This just works for our testing.
       # Won't work if the cluster name != host name
       {%- for entry in [
           'api',
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
index b6929fb887..4cbdee32fc 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/states/snakeoil_certs.sls
@@ -7,6 +7,8 @@
 {%- from "arvados/map.jinja" import arvados with context %}
 {%- set tpldir = curr_tpldir %}
 
+{%- set orig_cert_dir = salt['pillar.get']('extra_custom_certs_dir', '/srv/salt/certs')  %}
+
 include:
   - nginx.passenger
   - nginx.config
@@ -16,31 +18,49 @@ include:
 # we'll keep it simple here.
 {%- set arvados_ca_cert_file = '/etc/ssl/private/arvados-snakeoil-ca.pem' %}
 {%- set arvados_ca_key_file = '/etc/ssl/private/arvados-snakeoil-ca.key' %}
-{%- set arvados_cert_file = '/etc/ssl/private/arvados-snakeoil-cert.pem' %}
-{%- set arvados_csr_file = '/etc/ssl/private/arvados-snakeoil-cert.csr' %}
-{%- set arvados_key_file = '/etc/ssl/private/arvados-snakeoil-cert.key' %}
 
 {%- if grains.get('os_family') == 'Debian' %}
   {%- set arvados_ca_cert_dest = '/usr/local/share/ca-certificates/arvados-snakeoil-ca.crt' %}
   {%- set update_ca_cert = '/usr/sbin/update-ca-certificates' %}
   {%- set openssl_conf = '/etc/ssl/openssl.cnf' %}
+
+extra_snakeoil_certs_ssl_cert_pkg_installed:
+  pkg.installed:
+    - name: ssl-cert
+    - require_in:
+      - sls: postgres
+
 {%- else %}
   {%- set arvados_ca_cert_dest = '/etc/pki/ca-trust/source/anchors/arvados-snakeoil-ca.pem' %}
   {%- set update_ca_cert = '/usr/bin/update-ca-trust' %}
   {%- set openssl_conf = '/etc/pki/tls/openssl.cnf' %}
+
 {%- endif %}
 
-arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed:
+extra_snakeoil_certs_dependencies_pkg_installed:
   pkg.installed:
     - pkgs:
       - openssl
       - ca-certificates
 
-arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run:
+# Remove the RANDFILE parameter in openssl.cnf as it makes openssl fail in Ubuntu 18.04
+# Saving and restoring the rng state is not necessary anymore in the openssl 1.1.1
+# random generator, cf
+#   https://github.com/openssl/openssl/issues/7754
+#
+extra_snakeoil_certs_file_comment_etc_openssl_conf:
+  file.comment:
+    - name: /etc/ssl/openssl.cnf
+    - regex: ^RANDFILE.*
+    - onlyif: grep -q ^RANDFILE /etc/ssl/openssl.cnf
+    - require_in:
+      - cmd: extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run
+
+extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run:
   # Taken from https://github.com/arvados/arvados/blob/master/tools/arvbox/lib/arvbox/docker/service/certificate/run
   cmd.run:
     - name: |
-        # These dirs are not to CentOS-ish, but this is a helper script
+        # These dirs are not too CentOS-ish, but this is a helper script
         # and they should be enough
         mkdir -p /etc/ssl/certs/ /etc/ssl/private/ && \
         openssl req \
@@ -61,64 +81,55 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
       - test -f {{ arvados_ca_cert_file }}
       - openssl verify -CAfile {{ arvados_ca_cert_file }} {{ arvados_ca_cert_file }}
     - require:
-      - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
+      - pkg: extra_snakeoil_certs_dependencies_pkg_installed
 
-arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run:
+{%- set arvados_cert_file = orig_cert_dir ~ '/arvados-__HOSTNAME_EXT__.pem' %}
+{%- set arvados_csr_file = orig_cert_dir ~ '/arvadoos-__HOSTNAME_EXT__.csr' %}
+{%- set arvados_key_file = orig_cert_dir ~ '/arvados-__HOSTNAME_EXT__.key' %}
+
+extra_snakeoil_certs_arvados_snakeoil_cert___HOSTNAME_EXT___cmd_run:
   cmd.run:
     - name: |
-        cat > /tmp/openssl.cnf <<-CNF
+        cat > /tmp/__HOSTNAME_EXT__.openssl.cnf <<-CNF
         [req]
         default_bits = 2048
         prompt = no
         default_md = sha256
-        req_extensions = rext
         distinguished_name = dn
+        req_extensions = rext
+        [rext]
+        subjectAltName = @alt_names
         [dn]
         C   = CC
         ST  = Some State
         L   = Some Location
-        O   = Arvados Formula
-        OU  = arvados-formula
+        O   = Arvados Provision Example Single Host / Single Hostname
+        OU  = arvados-provision-example-single_host_single_hostname
         CN  = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
         emailAddress = admin@{{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
-        [rext]
-        subjectAltName = @alt_names
         [alt_names]
         {%- for entry in grains.get('ipv4') %}
         IP.{{ loop.index }} = {{ entry }}
         {%- endfor %}
-        {%- for entry in [
-            'keep',
-            'collections',
-            'download',
-            'keepweb',
-            'ws',
-            'workbench',
-            'workbench2',
-          ]
-        %}
-        DNS.{{ loop.index }} = {{ entry }}
-        {%- endfor %}
-        DNS.8 = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
-        DNS.9 = '__HOSTNAME_EXT__'
-        DNS.10 = '__HOSTNAME_INT__'
+        DNS.1 = {{ arvados.cluster.name }}.{{ arvados.cluster.domain }}
+        DNS.2 = '__HOSTNAME_EXT__'
         CNF
 
         # The req
         openssl req \
-          -config /tmp/openssl.cnf \
+          -config /tmp/__HOSTNAME_EXT__.openssl.cnf \
           -new \
           -nodes \
           -sha256 \
           -out {{ arvados_csr_file }} \
-          -keyout {{ arvados_key_file }} > /tmp/snake_oil_certs.output 2>&1 && \
+          -keyout {{ arvados_key_file }} > /tmp/snake_oil_certs.__HOSTNAME_EXT__.output 2>&1 && \
         # The cert
         openssl x509 \
           -req \
           -days 365 \
           -in {{ arvados_csr_file }} \
           -out {{ arvados_cert_file }} \
-          -extfile /tmp/openssl.cnf \
+          -extfile /tmp/__HOSTNAME_EXT__.openssl.cnf \
           -extensions rext \
           -CA {{ arvados_ca_cert_file }} \
           -CAkey {{ arvados_ca_key_file }} \
@@ -129,27 +140,19 @@ arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_c
       - test -f {{ arvados_key_file }}
       - openssl verify -CAfile {{ arvados_ca_cert_file }} {{ arvados_cert_file }}
     - require:
-      - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_dependencies_pkg_installed
-      - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_ca_cmd_run
-    # We need this before we can add the nginx's snippet
+      - pkg: extra_snakeoil_certs_dependencies_pkg_installed
+      - cmd: extra_snakeoil_certs_arvados_snakeoil_ca_cmd_run
     - require_in:
-      - file: nginx_snippet_arvados-snakeoil.conf
+      - file: extra_custom_certs_file_copy_arvados-__HOSTNAME_EXT__.pem
+      - file: extra_custom_certs_file_copy_arvados-__HOSTNAME_EXT__.key
 
-{%- if grains.get('os_family') == 'Debian' %}
-arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed:
-  pkg.installed:
-    - name: ssl-cert
-    - require_in:
-      - sls: postgres
-
-arvados_test_salt_states_examples_single_host_snakeoil_certs_certs_permissions_cmd_run:
+  {%- if grains.get('os_family') == 'Debian' %}
+extra_snakeoil_certs_certs_permissions___HOSTNAME_EXT___cmd_run:
   file.managed:
     - name: {{ arvados_key_file }}
     - owner: root
     - group: ssl-cert
     - require:
-      - cmd: arvados_test_salt_states_examples_single_host_snakeoil_certs_arvados_snake_oil_cert_cmd_run
-      - pkg: arvados_test_salt_states_examples_single_host_snakeoil_certs_ssl_cert_pkg_installed
-    - require_in:
-      - file: nginx_snippet_arvados-snakeoil.conf
-{%- endif %}
+      - cmd: extra_snakeoil_certs_arvados_snakeoil_cert___HOSTNAME_EXT___cmd_run
+      - pkg: extra_snakeoil_certs_ssl_cert_pkg_installed
+  {%- endif %}
diff --git a/tools/salt-install/local.params.example.multiple_hosts b/tools/salt-install/local.params.example.multiple_hosts
index cb0afecc46..32d1f8bb96 100644
--- a/tools/salt-install/local.params.example.multiple_hosts
+++ b/tools/salt-install/local.params.example.multiple_hosts
@@ -58,14 +58,14 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 DATABASE_PASSWORD=please_set_this_to_some_secure_value
 
 # SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will fail
-# to communicate and can silently drop traffic. You can try to use the Letsencrypt
-# salt formula (https://github.com/saltstack-formulas/letsencrypt-formula) to try to
-# automatically obtain and install SSL certificates for your instances or set this
-# variable to "no", provide and upload your own certificates to the instances and
-# modify the 'nginx_*' salt pillars accordingly (see CUSTOM_CERTS_DIR below)
-USE_LETSENCRYPT="yes"
-USE_LETSENCRYPT_IAM_USER="yes"
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
+#
+# See https://doc.arvados.org/intall/salt-multi-host.html for more information.
+SSL_MODE="lets-encrypt"
+USE_LETSENCRYPT_ROUTE53="yes"
 # For collections, we need to obtain a wildcard certificate for
 # '*.collections..'. This is only possible through a DNS-01 challenge.
 # For that reason, you'll need to provide AWS credentials with permissions to manage
@@ -76,7 +76,7 @@ LE_AWS_ACCESS_KEY_ID="AKIABCDEFGHIJKLMNOPQ"
 LE_AWS_SECRET_ACCESS_KEY="thisistherandomstringthatisyoursecretkey"
 
 # If you going to provide your own certificates for Arvados, the provision script can
-# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
+# help you deploy them. In order to do that, you need to set `SSL_MODE=bring-your-own` above,
 # and copy the required certificates under the directory specified in the next line.
 # The certs will be copied from this directory by the provision script.
 # Please set it to the FULL PATH to the certs dir if you're going to use a different dir
@@ -119,8 +119,8 @@ RELEASE="production"
 
 # Formulas versions
 # ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_multiple_hostnames b/tools/salt-install/local.params.example.single_host_multiple_hostnames
index ef47467e57..d6bfb102e9 100644
--- a/tools/salt-install/local.params.example.single_host_multiple_hostnames
+++ b/tools/salt-install/local.params.example.single_host_multiple_hostnames
@@ -11,13 +11,9 @@ CLUSTER="cluster_fixme_or_this_wont_work"
 # The domainname you want tou give to your cluster's hosts
 DOMAIN="domain_fixme_or_this_wont_work"
 
-# Host SSL port where you want to point your browser to access Arvados
-# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
-# You can point it to another port if desired
-# In Vagrant, make sure it matches what you set in the Vagrantfile (8443)
+# External ports used by the Arvados services
 CONTROLLER_EXT_SSL_PORT=443
 KEEP_EXT_SSL_PORT=25101
-# Both for collections and downloads
 KEEPWEB_EXT_SSL_PORT=9002
 WEBSHELL_EXT_SSL_PORT=4202
 WEBSOCKET_EXT_SSL_PORT=8002
@@ -25,7 +21,6 @@ WORKBENCH1_EXT_SSL_PORT=443
 WORKBENCH2_EXT_SSL_PORT=3001
 
 INITIAL_USER="admin"
-
 # If not specified, the initial user email will be composed as
 # INITIAL_USER@CLUSTER.DOMAIN
 INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work"
@@ -40,42 +35,22 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 DATABASE_PASSWORD=please_set_this_to_some_secure_value
 
 # SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will
-# fail to communicate and can silently drop traffic. Set USE_LETSENCRYPT="yes"
-# to use the Let's Encrypt salt formula
-# (https://github.com/saltstack-formulas/letsencrypt-formula) to automatically
-# obtain and install SSL certificates for your hostname(s).
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
 #
-# Alternatively, set this variable to "no" and provide and upload your own
-# certificates to the instances and modify the 'nginx_*' salt pillars
-# accordingly
-USE_LETSENCRYPT="no"
+# See https://doc.arvados.org/intall/salt-single-host.html#certificates for more information.
+SSL_MODE="self-signed"
 
-# If you going to provide your own certificates for Arvados, the provision script can
-# help you deploy them. In order to do that, you need to set `USE_LETSENCRYPT=no` above,
-# and copy the required certificates under the directory specified in the next line.
-# The certs will be copied from this directory by the provision script.
-# Please set it to the FULL PATH to the certs dir if you're going to use a different dir
-# Default is "${SCRIPT_DIR}/certs", where the variable "SCRIPT_DIR" has the path to the
-# directory where the  "provision.sh" script was copied in the destination host.
+# CUSTOM_CERTS_DIR is only used when SSL_MODE is set to "bring-your-own".
+# See https://doc.arvados.org/intall/salt-single-host.html#bring-your-own for more information.
 # CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
-# The script expects cert/key files with these basenames (matching the role except for
-# keepweb, which is split in both download/collections):
-#  "controller"
-#  "websocket"
-#  "workbench"
-#  "workbench2"
-#  "webshell"
-#  "download"         # Part of keepweb
-#  "collections"      # Part of keepweb
-#  "keepproxy"
-# Ie., 'keepproxy', the script will lookup for
-# ${CUSTOM_CERTS_DIR}/keepproxy.crt
-# ${CUSTOM_CERTS_DIR}/keepproxy.key
 
 # The directory to check for the config files (pillars, states) you want to use.
 # There are a few examples under 'config_examples'.
 # CONFIG_DIR="local_config_dir"
+
 # Extra states to apply. If you use your own subdir, change this value accordingly
 # EXTRA_STATES_DIR="${CONFIG_DIR}/states"
 
@@ -95,8 +70,8 @@ RELEASE="production"
 
 # Formulas versions
 # ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/local.params.example.single_host_single_hostname b/tools/salt-install/local.params.example.single_host_single_hostname
index d09cdb2ef0..b6c7e5f7a5 100644
--- a/tools/salt-install/local.params.example.single_host_single_hostname
+++ b/tools/salt-install/local.params.example.single_host_single_hostname
@@ -5,42 +5,36 @@
 
 # These are the basic parameters to configure the installation
 
-# The FIVE ALPHANUMERIC CHARACTERS name you want to give your cluster
+# The Arvados cluster ID, needs to be five alphanumeric characters.
 CLUSTER="cluster_fixme_or_this_wont_work"
 
-# The domainname you want tou give to your cluster's hosts
+# The domainname for your cluster's hosts
 DOMAIN="domain_fixme_or_this_wont_work"
 
-# Set this value when installing a cluster in a single host with a single hostname
-# to access all the instances. Not used in the other examples.
-# When using virtualization (ie AWS), this should be
-# the EXTERNAL/PUBLIC hostname for the instance.
-# If empty, ${CLUSTER}.${DOMAIN} will be used
-HOSTNAME_EXT=""
-# The internal hostname for the host. In the example files, only used in the
-# single_host/single_hostname example
-HOSTNAME_INT="127.0.1.1"
-# Host SSL port where you want to point your browser to access Arvados
-# Defaults to 443 for regular runs, and to 8443 when called in Vagrant.
-# You can point it to another port if desired
-# In Vagrant, make sure it matches what you set in the Vagrantfile (8443)
-CONTROLLER_EXT_SSL_PORT=9443
-KEEP_EXT_SSL_PORT=35101
-# Both for collections and downloads
-KEEPWEB_EXT_SSL_PORT=11002
-WEBSHELL_EXT_SSL_PORT=14202
-WEBSOCKET_EXT_SSL_PORT=18002
-WORKBENCH1_EXT_SSL_PORT=9444
-WORKBENCH2_EXT_SSL_PORT=9445
+# Set this value when installing a cluster in a single host with a single
+# hostname to access all the instances. HOSTNAME_EXT should be set to the
+# external hostname for the instance.
+HOSTNAME_EXT="hostname_ext_fixme_or_this_wont_work"
 
-INITIAL_USER="admin"
+# The internal IP address for the host.
+IP_INT="ip_int_fixme_or_this_wont_work"
+
+# External ports used by the Arvados services
+CONTROLLER_EXT_SSL_PORT=8800
+KEEP_EXT_SSL_PORT=8801
+KEEPWEB_EXT_SSL_PORT=8802
+WEBSHELL_EXT_SSL_PORT=8803
+WEBSOCKET_EXT_SSL_PORT=8804
+WORKBENCH1_EXT_SSL_PORT=8805
+WORKBENCH2_EXT_SSL_PORT=443
 
+INITIAL_USER="admin"
 # If not specified, the initial user email will be composed as
 # INITIAL_USER@CLUSTER.DOMAIN
 INITIAL_USER_EMAIL="admin@cluster_fixme_or_this_wont_work.domain_fixme_or_this_wont_work"
 INITIAL_USER_PASSWORD="password"
 
-# YOU SHOULD CHANGE THESE TO SOME RANDOM STRINGS
+# Populate these values with random strings
 BLOB_SIGNING_KEY=blobsigningkeymushaveatleast32characters
 MANAGEMENT_TOKEN=managementtokenmushaveatleast32characters
 SYSTEM_ROOT_TOKEN=systemroottokenmushaveatleast32characters
@@ -49,20 +43,22 @@ WORKBENCH_SECRET_KEY=workbenchsecretkeymushaveatleast32characters
 DATABASE_PASSWORD=please_set_this_to_some_secure_value
 
 # SSL CERTIFICATES
-# Arvados REQUIRES valid SSL to work correctly. Otherwise, some components will
-# fail to communicate and can silently drop traffic. Set USE_LETSENCRYPT="yes"
-# to use the Let's Encrypt salt formula
-# (https://github.com/saltstack-formulas/letsencrypt-formula) to automatically
-# obtain and install SSL certificates for your hostname(s).
+# Arvados requires SSL certificates to work correctly. This installer supports these options:
+# * self-signed: let the installer create self-signed certificate(s)
+# * bring-your-own: supply your own certificate(s) in the `certs` directory
+# * lets-encrypt: automatically obtain and install SSL certificates for your hostname(s)
 #
-# Alternatively, set this variable to "no" and provide and upload your own
-# certificates to the instances and modify the 'nginx_*' salt pillars
-# accordingly
-USE_LETSENCRYPT="no"
+# See https://doc.arvados.org/intall/salt-single-host.html#certificates for more information.
+SSL_MODE="self-signed"
+
+# CUSTOM_CERTS_DIR is only used when SSL_MODE is set to "bring-your-own".
+# See https://doc.arvados.org/intall/salt-single-host.html#bring-your-own for more information.
+# CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
 
 # The directory to check for the config files (pillars, states) you want to use.
 # There are a few examples under 'config_examples'.
 # CONFIG_DIR="local_config_dir"
+
 # Extra states to apply. If you use your own subdir, change this value accordingly
 # EXTRA_STATES_DIR="${CONFIG_DIR}/states"
 
@@ -82,8 +78,8 @@ RELEASE="production"
 
 # Formulas versions
 # ARVADOS_TAG="2.2.0"
-# POSTGRES_TAG="v0.43.0"
-# NGINX_TAG="temp-fix-missing-statements-in-pillar"
-# DOCKER_TAG="v2.0.7"
+# POSTGRES_TAG="v0.44.0"
+# NGINX_TAG="v2.8.1"
+# DOCKER_TAG="v2.4.2"
 # LOCALE_TAG="v0.3.4"
 # LETSENCRYPT_TAG="v2.1.0"
diff --git a/tools/salt-install/provision.sh b/tools/salt-install/provision.sh
index 594dad2ebb..0f3c9a1411 100755
--- a/tools/salt-install/provision.sh
+++ b/tools/salt-install/provision.sh
@@ -29,6 +29,7 @@ usage() {
   echo >&2 "                                                controller"
   echo >&2 "                                                dispatcher"
   echo >&2 "                                                keepproxy"
+  echo >&2 "                                                keepbalance"
   echo >&2 "                                                keepstore"
   echo >&2 "                                                keepweb"
   echo >&2 "                                                shell"
@@ -107,7 +108,7 @@ arguments() {
         for i in ${2//,/ }
           do
             # Verify the role exists
-            if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
+            if [[ ! "database,api,controller,keepstore,websocket,keepweb,workbench2,webshell,keepbalance,keepproxy,shell,workbench,dispatcher" == *"$i"* ]]; then
               echo "The role '${i}' is not a valid role"
               usage
               exit 1
@@ -164,12 +165,13 @@ LOG_LEVEL="info"
 CONTROLLER_EXT_SSL_PORT=443
 TESTS_DIR="tests"
 
+NGINX_INSTALL_SOURCE="install_from_repo"
+
 CLUSTER=""
 DOMAIN=""
 
 # Hostnames/IPs used for single-host deploys
-HOSTNAME_EXT=""
-HOSTNAME_INT="127.0.1.1"
+IP_INT="127.0.1.1"
 
 # Initial user setup
 INITIAL_USER=""
@@ -185,7 +187,8 @@ WEBSOCKET_EXT_SSL_PORT=8002
 WORKBENCH1_EXT_SSL_PORT=443
 WORKBENCH2_EXT_SSL_PORT=3001
 
-USE_LETSENCRYPT="no"
+SSL_MODE="self-signed"
+USE_LETSENCRYPT_ROUTE53="no"
 CUSTOM_CERTS_DIR="${SCRIPT_DIR}/certs"
 
 ## These are ARVADOS-related parameters
@@ -204,9 +207,9 @@ VERSION="latest"
 # BRANCH="main"
 
 # Other formula versions we depend on
-POSTGRES_TAG="v0.43.0"
-NGINX_TAG="temp-fix-missing-statements-in-pillar"
-DOCKER_TAG="v2.0.7"
+POSTGRES_TAG="v0.44.0"
+NGINX_TAG="v2.8.1"
+DOCKER_TAG="v2.4.2"
 LOCALE_TAG="v0.3.4"
 LETSENCRYPT_TAG="v2.1.0"
 
@@ -254,7 +257,21 @@ if ! grep -qE '^[[:alnum:]]{5}$' <<<${CLUSTER} ; then
 fi
 
 # Only used in single_host/single_name deploys
-if [ "x${HOSTNAME_EXT}" = "x" ] ; then
+if [ ! -z "${HOSTNAME_EXT}" ] ; then
+  # We need to add some extra control vars to manage a single certificate vs. multiple
+  USE_SINGLE_HOSTNAME="yes"
+  # Make sure that the value configured as IP_INT is a real IP on the system.
+  # If we don't error out early here when there is a mismatch, the formula will
+  # fail with hard to interpret nginx errors later on.
+  ip addr list |grep -q "${IP_INT}/"
+  if [[ $? -ne 0 ]]; then
+    echo "Unable to find the IP_INT address '${IP_INT}' on the system, please correct the value in local.params. Exiting..."
+    exit 1
+  fi
+else
+  USE_SINGLE_HOSTNAME="no"
+  # We set this variable, anyway, so sed lines do not fail and we don't need to add more
+  # conditionals
   HOSTNAME_EXT="${CLUSTER}.${DOMAIN}"
 fi
 
@@ -269,7 +286,7 @@ else
   case ${OS_ID} in
     "centos")
       echo "WARNING! Disabling SELinux, see https://dev.arvados.org/issues/18019"
-      sed -i 's/SELINUX=enforcing/SELINUX=permissive' /etc/sysconfig/selinux
+      sed -i 's/SELINUX=enforcing/SELINUX=permissive/g' /etc/sysconfig/selinux
       setenforce permissive
       yum install -y  curl git jq
       ;;
@@ -313,18 +330,23 @@ rm -rf ${F_DIR}/* || exit 1
 git clone --quiet https://github.com/saltstack-formulas/docker-formula.git ${F_DIR}/docker
 ( cd docker && git checkout --quiet tags/"${DOCKER_TAG}" -b "${DOCKER_TAG}" )
 
+echo "...locale"
 git clone --quiet https://github.com/saltstack-formulas/locale-formula.git ${F_DIR}/locale
 ( cd locale && git checkout --quiet tags/"${LOCALE_TAG}" -b "${LOCALE_TAG}" )
 
-git clone --quiet https://github.com/netmanagers/nginx-formula.git ${F_DIR}/nginx
+echo "...nginx"
+git clone --quiet https://github.com/saltstack-formulas/nginx-formula.git ${F_DIR}/nginx
 ( cd nginx && git checkout --quiet tags/"${NGINX_TAG}" -b "${NGINX_TAG}" )
 
+echo "...postgres"
 git clone --quiet https://github.com/saltstack-formulas/postgres-formula.git ${F_DIR}/postgres
 ( cd postgres && git checkout --quiet tags/"${POSTGRES_TAG}" -b "${POSTGRES_TAG}" )
 
+echo "...letsencrypt"
 git clone --quiet https://github.com/saltstack-formulas/letsencrypt-formula.git ${F_DIR}/letsencrypt
 ( cd letsencrypt && git checkout --quiet tags/"${LETSENCRYPT_TAG}" -b "${LETSENCRYPT_TAG}" )
 
+echo "...arvados"
 git clone --quiet https://git.arvados.org/arvados-formula.git ${F_DIR}/arvados
 
 # If we want to try a specific branch of the formula
@@ -361,7 +383,7 @@ for f in $(ls "${SOURCE_PILLARS_DIR}"/*); do
        s#__CLUSTER__#${CLUSTER}#g;
        s#__DOMAIN__#${DOMAIN}#g;
        s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
-       s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
+       s#__IP_INT__#${IP_INT}#g;
        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
        s#__INITIAL_USER__#${INITIAL_USER}#g;
@@ -402,16 +424,21 @@ fi
 mkdir -p ${T_DIR}
 # Replace cluster and domain name in the test files
 for f in $(ls "${SOURCE_TESTS_DIR}"/*); do
-  sed "s#__CLUSTER__#${CLUSTER}#g;
+  FILTERS="s#__CLUSTER__#${CLUSTER}#g;
        s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
        s#__DOMAIN__#${DOMAIN}#g;
-       s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
+       s#__IP_INT__#${IP_INT}#g;
        s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
        s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g
        s#__INITIAL_USER__#${INITIAL_USER}#g;
        s#__DATABASE_PASSWORD__#${DATABASE_PASSWORD}#g;
-       s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g" \
-  "${f}" > ${T_DIR}/$(basename "${f}")
+       s#__SYSTEM_ROOT_TOKEN__#${SYSTEM_ROOT_TOKEN}#g"
+  if [ "$USE_SINGLE_HOSTNAME" = "yes" ]; then
+    FILTERS="s#__CLUSTER__.__DOMAIN__#${HOSTNAME_EXT}#g;
+       $FILTERS"
+  fi
+  sed "$FILTERS" \
+    "${f}" > ${T_DIR}/$(basename "${f}")
 done
 chmod 755 ${T_DIR}/run-test.sh
 
@@ -426,7 +453,7 @@ if [ -d "${SOURCE_STATES_DIR}" ]; then
          s#__CONTROLLER_EXT_SSL_PORT__#${CONTROLLER_EXT_SSL_PORT}#g;
          s#__DOMAIN__#${DOMAIN}#g;
          s#__HOSTNAME_EXT__#${HOSTNAME_EXT}#g;
-         s#__HOSTNAME_INT__#${HOSTNAME_INT}#g;
+         s#__IP_INT__#${IP_INT}#g;
          s#__INITIAL_USER_EMAIL__#${INITIAL_USER_EMAIL}#g;
          s#__INITIAL_USER_PASSWORD__#${INITIAL_USER_PASSWORD}#g;
          s#__INITIAL_USER__#${INITIAL_USER}#g;
@@ -449,6 +476,7 @@ if [ -d "${SOURCE_STATES_DIR}" ]; then
          s#__WORKBENCH2_INT_IP__#${WORKBENCH2_INT_IP}#g;
          s#__DATABASE_INT_IP__#${DATABASE_INT_IP}#g;
          s#__WEBSHELL_EXT_SSL_PORT__#${WEBSHELL_EXT_SSL_PORT}#g;
+         s#__SHELL_INT_IP__#${SHELL_INT_IP}#g;
          s#__WEBSOCKET_EXT_SSL_PORT__#${WEBSOCKET_EXT_SSL_PORT}#g;
          s#__WORKBENCH1_EXT_SSL_PORT__#${WORKBENCH1_EXT_SSL_PORT}#g;
          s#__WORKBENCH2_EXT_SSL_PORT__#${WORKBENCH2_EXT_SSL_PORT}#g;
@@ -478,18 +506,19 @@ EOFPSLS
 
 # States, extra states
 if [ -d "${F_DIR}"/extra/extra ]; then
-  if [ "$DEV_MODE" = "yes" ]; then
+  SKIP_SNAKE_OIL="snakeoil_certs"
+
+  if [[ "$DEV_MODE" = "yes" || "${SSL_MODE}" == "self-signed" ]] ; then
     # In dev mode, we create some snake oil certs that we'll
-    # use as CUSTOM_CERTS, so we don't skip the states file
-    SKIP_SNAKE_OIL="dont_snakeoil_certs"
-  else
-    SKIP_SNAKE_OIL="snakeoil_certs"
+    # use as CUSTOM_CERTS, so we don't skip the states file.
+    # Same when using self-signed certificates.
+    SKIP_SNAKE_OIL="dont_add_snakeoil_certs"
   fi
   for f in $(ls "${F_DIR}"/extra/extra/*.sls | grep -v ${SKIP_SNAKE_OIL}); do
   echo "    - extra.$(basename ${f} | sed 's/.sls$//g')" >> ${S_DIR}/top.sls
   done
-  # Use custom certs
-  if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+  # Use byo or self-signed certificates
+  if [ "${SSL_MODE}" != "lets-encrypt" ]; then
     mkdir -p "${F_DIR}"/extra/extra/files
   fi
 fi
@@ -499,14 +528,13 @@ fi
 if [ -z "${ROLES}" ]; then
   # States
   echo "    - nginx.passenger" >> ${S_DIR}/top.sls
-  # Currently, only available on config_examples/multi_host/aws
-  if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-    if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+  if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+    if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
       grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - extra.aws_credentials" >> ${S_DIR}/top.sls
     fi
     grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
   else
-    # Use custom certs
+    # Use custom certs, as both bring-your-own and self-signed are copied using this state
     # Copy certs to formula extra/files
     # In dev mode, the files will be created and put in the destination directory by the
     # snakeoil_certs.sls state file
@@ -533,12 +561,15 @@ if [ -z "${ROLES}" ]; then
   echo "    - nginx_workbench_configuration" >> ${P_DIR}/top.sls
   echo "    - postgresql" >> ${P_DIR}/top.sls
 
-  # Currently, only available on config_examples/multi_host/aws
-  if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-    if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+  # We need to tweak the Nginx's pillar depending whether we want plan nginx or nginx+passenger
+  NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+  sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
+
+  if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+    if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
       grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
     fi
-    grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
+    grep -q "letsencrypt" ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
 
     # As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
     for c in controller websocket workbench workbench2 webshell download collections keepproxy; do
@@ -554,24 +585,37 @@ if [ -z "${ROLES}" ]; then
     echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
     echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
 
-    for c in controller websocket workbench workbench2 webshell download collections keepproxy; do
-      grep -q ${c} ${P_DIR}/extra_custom_certs.sls || echo "  - ${c}" >> ${P_DIR}/extra_custom_certs.sls
-
-      # As the pillar differ whether we use LE or custom certs, we need to do a final edition on them
-      sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${c}.pem/g;
-              s#__CERT_PEM__#/etc/nginx/ssl/arvados-${c}.pem#g;
-              s#__CERT_KEY__#/etc/nginx/ssl/arvados-${c}.key#g" \
+    for c in controller websocket workbench workbench2 webshell keepweb keepproxy; do
+      # Are we in a single-host-single-hostname env?
+      if [ "${USE_SINGLE_HOSTNAME}" = "yes" ]; then
+        # Are we in a single-host-single-hostname env?
+        CERT_NAME=${HOSTNAME_EXT}
+      else
+        # We are in a multiple-hostnames env
+        CERT_NAME=${c}
+      fi
+
+      if [[ "$SSL_MODE" == "bring-your-own" ]]; then
+        copy_custom_cert ${CUSTOM_CERTS_DIR} ${CERT_NAME}
+      fi
+
+      grep -q ${CERT_NAME} ${P_DIR}/extra_custom_certs.sls || echo "  - ${CERT_NAME}" >> ${P_DIR}/extra_custom_certs.sls
+
+      # As the pillar differs whether we use LE or custom certs, we need to do a final edition on them
+      sed -i "s/__CERT_REQUIRES__/file: extra_custom_certs_file_copy_arvados-${CERT_NAME}.pem/g;
+              s#__CERT_PEM__#/etc/nginx/ssl/arvados-${CERT_NAME}.pem#g;
+              s#__CERT_KEY__#/etc/nginx/ssl/arvados-${CERT_NAME}.key#g" \
       ${P_DIR}/nginx_${c}_configuration.sls
     done
   fi
 else
   # If we add individual roles, make sure we add the repo first
   echo "    - arvados.repo" >> ${S_DIR}/top.sls
-  # We add the custom_certs state
-  grep -q "custom_certs"    ${S_DIR}/top.sls || echo "    - extra.custom_certs" >> ${S_DIR}/top.sls
+  # We add the extra_custom_certs state
+  grep -q "extra.custom_certs"    ${S_DIR}/top.sls || echo "    - extra.custom_certs" >> ${S_DIR}/top.sls
 
   # And we add the basic part for the certs pillar
-  if [ "x${USE_LETSENCRYPT}" != "xyes" ]; then
+  if [ "${SSL_MODE}" != "lets-encrypt" ]; then
     # And add the certs in the custom_certs pillar
     echo "extra_custom_certs_dir: /srv/salt/certs" > ${P_DIR}/extra_custom_certs.sls
     echo "extra_custom_certs:" >> ${P_DIR}/extra_custom_certs.sls
@@ -590,18 +634,23 @@ else
         # States
         # FIXME: https://dev.arvados.org/issues/17352
         grep -q "postgres.client" ${S_DIR}/top.sls || echo "    - postgres.client" >> ${S_DIR}/top.sls
-        grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
+        if grep -q "    - nginx.*$" ${S_DIR}/top.sls; then
+          sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${S_DIR}/top.sls
+        else
+          echo "    - nginx.passenger" >> ${S_DIR}/top.sls
+        fi
         ### If we don't install and run LE before arvados-api-server, it fails and breaks everything
         ### after it. So we add this here as we are, after all, sharing the host for api and controller
-        # Currently, only available on config_examples/multi_host/aws
-        if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-          if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+        if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+          if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
           fi
           grep -q "letsencrypt" ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
         else
           # Use custom certs
-          copy_custom_cert ${CUSTOM_CERTS_DIR} controller
+          if [ "${SSL_MODE}" = "bring-your-own" ]; then
+            copy_custom_cert ${CUSTOM_CERTS_DIR} controller
+          fi
           grep -q controller ${P_DIR}/extra_custom_certs.sls || echo "  - controller" >> ${P_DIR}/extra_custom_certs.sls
         fi
         grep -q "arvados.${R}" ${S_DIR}/top.sls    || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
@@ -610,23 +659,39 @@ else
         grep -q "postgresql" ${P_DIR}/top.sls               || echo "    - postgresql" >> ${P_DIR}/top.sls
         grep -q "nginx_passenger" ${P_DIR}/top.sls          || echo "    - nginx_passenger" >> ${P_DIR}/top.sls
         grep -q "nginx_${R}_configuration" ${P_DIR}/top.sls || echo "    - nginx_${R}_configuration" >> ${P_DIR}/top.sls
+
+        # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
+        NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+        sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
       ;;
       "controller" | "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
         # States
-        grep -q "nginx.passenger" ${S_DIR}/top.sls || echo "    - nginx.passenger" >> ${S_DIR}/top.sls
-        # Currently, only available on config_examples/multi_host/aws
-        if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-          if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+        if [ "${R}" = "workbench" ]; then
+          NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
+          if grep -q "    - nginx$" ${S_DIR}/top.sls; then
+            sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${S_DIR}/top.sls
+          else
+            echo "    - nginx.passenger" >> ${S_DIR}/top.sls
+          fi
+        else
+          grep -q "nginx" ${S_DIR}/top.sls || echo "    - nginx" >> ${S_DIR}/top.sls
+        fi
+        if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+          if [ "x${USE_LETSENCRYPT_ROUTE53}" = "xyes" ]; then
             grep -q "aws_credentials" ${S_DIR}/top.sls || echo "    - aws_credentials" >> ${S_DIR}/top.sls
           fi
           grep -q "letsencrypt"     ${S_DIR}/top.sls || echo "    - letsencrypt" >> ${S_DIR}/top.sls
         else
           # Use custom certs, special case for keepweb
           if [ ${R} = "keepweb" ]; then
-            copy_custom_cert ${CUSTOM_CERTS_DIR} download
-            copy_custom_cert ${CUSTOM_CERTS_DIR} collections
+            if [ "${SSL_MODE}" = "bring-your-own" ]; then
+              copy_custom_cert ${CUSTOM_CERTS_DIR} download
+              copy_custom_cert ${CUSTOM_CERTS_DIR} collections
+            fi
           else
-            copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
+            if [ "${SSL_MODE}" = "bring-your-own" ]; then
+              copy_custom_cert ${CUSTOM_CERTS_DIR} ${R}
+            fi
           fi
         fi
         # webshell role is just a nginx vhost, so it has no state
@@ -642,9 +707,8 @@ else
           grep -q "nginx_collections_configuration" ${P_DIR}/top.sls || echo "    - nginx_collections_configuration" >> ${P_DIR}/top.sls
         fi
 
-        # Currently, only available on config_examples/multi_host/aws
-        if [ "x${USE_LETSENCRYPT}" = "xyes" ]; then
-          if [ "x${USE_LETSENCRYPT_IAM_USER}" != "xyes" ]; then
+        if [ "${SSL_MODE}" = "lets-encrypt" ]; then
+          if [ "${USE_LETSENCRYPT_ROUTE53}" = "yes" ]; then
             grep -q "aws_credentials" ${P_DIR}/top.sls || echo "    - aws_credentials" >> ${P_DIR}/top.sls
           fi
           grep -q "letsencrypt"     ${P_DIR}/top.sls || echo "    - letsencrypt" >> ${P_DIR}/top.sls
@@ -684,6 +748,8 @@ else
             grep -q ${R} ${P_DIR}/extra_custom_certs.sls || echo "  - ${R}" >> ${P_DIR}/extra_custom_certs.sls
           fi
         fi
+        # We need to tweak the Nginx's pillar depending whether we want plain nginx or nginx+passenger
+        sed -i "s/__NGINX_INSTALL_SOURCE__/${NGINX_INSTALL_SOURCE}/g" ${P_DIR}/nginx_passenger.sls
       ;;
       "shell")
         # States
@@ -692,13 +758,7 @@ else
         # Pillars
         grep -q "docker" ${P_DIR}/top.sls       || echo "    - docker" >> ${P_DIR}/top.sls
       ;;
-      "dispatcher")
-        # States
-        grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
-        # Pillars
-        # ATM, no specific pillar needed
-      ;;
-      "keepstore")
+      "dispatcher" | "keepbalance" | "keepstore")
         # States
         grep -q "arvados.${R}" ${S_DIR}/top.sls || echo "    - arvados.${R}" >> ${S_DIR}/top.sls
         # Pillars
@@ -717,33 +777,14 @@ if [ "${DUMP_CONFIG}" = "yes" ]; then
   exit 0
 fi
 
-# FIXME! #16992 Temporary fix for psql call in arvados-api-server
-if [ -e /root/.psqlrc ]; then
-  if ! ( grep 'pset pager off' /root/.psqlrc ); then
-    RESTORE_PSQL="yes"
-    cp /root/.psqlrc /root/.psqlrc.provision.backup
-  fi
-else
-  DELETE_PSQL="yes"
-fi
-
-echo '\pset pager off' >> /root/.psqlrc
-# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
-
 # Now run the install
 salt-call --local state.apply -l ${LOG_LEVEL}
 
-# FIXME! #16992 Temporary fix for psql call in arvados-api-server
-if [ "x${DELETE_PSQL}" = "xyes" ]; then
-  echo "Removing .psql file"
-  rm /root/.psqlrc
-fi
-
-if [ "x${RESTORE_PSQL}" = "xyes" ]; then
-  echo "Restoring .psql file"
-  mv -v /root/.psqlrc.provision.backup /root/.psqlrc
+# Finally, make sure that /etc/hosts is not overwritten on reboot
+if [ -d /etc/cloud/cloud.cfg.d ]; then
+  # TODO: will this work on CentOS?
+  sed -i 's/^manage_etc_hosts: true/#manage_etc_hosts: true/g' /etc/cloud/cloud.cfg.d/*
 fi
-# END FIXME! #16992 Temporary fix for psql call in arvados-api-server
 
 # Leave a copy of the Arvados CA so the user can copy it where it's required
 if [ "$DEV_MODE" = "yes" ]; then
diff --git a/tools/salt-install/tests/run-test.sh b/tools/salt-install/tests/run-test.sh
index a47294b3bd..cf43273a14 100755
--- a/tools/salt-install/tests/run-test.sh
+++ b/tools/salt-install/tests/run-test.sh
@@ -37,10 +37,6 @@ fi
 
 echo "Arvados project uuid is '${project_uuid}'"
 
-echo "Uploading arvados/jobs' docker image to the project"
-VERSION="2.1.1"
-arv-keepdocker --pull arvados/jobs "${VERSION}" --project-uuid "${project_uuid}"
-
 # Create the initial user
 echo "Creating initial user '__INITIAL_USER__'"
 user_uuid=$(arv --format=uuid user list --filters '[["email", "=", "__INITIAL_USER_EMAIL__"], ["username", "=", "__INITIAL_USER__"]]')