From: Tom Clegg Date: Mon, 4 Dec 2023 19:35:36 +0000 (-0500) Subject: Merge branch '21217-invalid-auth-header' X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/bfbf93b7ae6bd90ca8417fed848bad882aba023c?hp=43b47017a9eb5e2b427c34c12c8e2344bef6f402 Merge branch '21217-invalid-auth-header' refs #21217 Arvados-DCO-1.1-Signed-off-by: Tom Clegg --- diff --git a/build/package-build-dockerfiles/Makefile b/build/package-build-dockerfiles/Makefile index b2f2b0cc1b..1d8066312b 100644 --- a/build/package-build-dockerfiles/Makefile +++ b/build/package-build-dockerfiles/Makefile @@ -3,32 +3,47 @@ # SPDX-License-Identifier: AGPL-3.0 SHELL := '/bin/bash' -all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated +all: centos7/generated centos7/generated: common-generated-all test -d centos7/generated || mkdir centos7/generated cp -f -rlt centos7/generated common-generated/* +all: debian10/generated debian10/generated: common-generated-all test -d debian10/generated || mkdir debian10/generated cp -f -rlt debian10/generated common-generated/* +all: debian11/generated debian11/generated: common-generated-all test -d debian11/generated || mkdir debian11/generated cp -f -rlt debian11/generated common-generated/* +all: debian12/generated +debian12/generated: common-generated-all + test -d debian12/generated || mkdir debian12/generated + cp -f -rlt debian12/generated common-generated/* + +all: rocky8/generated rocky8/generated: common-generated-all test -d rocky8/generated || mkdir rocky8/generated cp -f -rlt rocky8/generated common-generated/* +all: ubuntu1804/generated ubuntu1804/generated: common-generated-all test -d ubuntu1804/generated || mkdir ubuntu1804/generated cp -f -rlt ubuntu1804/generated common-generated/* +all: ubuntu2004/generated ubuntu2004/generated: common-generated-all test -d ubuntu2004/generated || mkdir ubuntu2004/generated cp -f -rlt ubuntu2004/generated common-generated/* +all: ubuntu2204/generated +ubuntu2204/generated: common-generated-all + test -d ubuntu2204/generated || mkdir ubuntu2204/generated + cp -f -rlt ubuntu2204/generated common-generated/* + GOTARBALL_=DOES_NOT_EXIST NODETARBALL_=DOES_NOT_EXIST GOVERSION=$(shell grep 'const goversion =' ../../lib/install/deps.go |awk -F'"' '{print $$2}') diff --git a/build/package-build-dockerfiles/centos7/Dockerfile b/build/package-build-dockerfiles/centos7/Dockerfile index f731f1a426..a423819dd0 100644 --- a/build/package-build-dockerfiles/centos7/Dockerfile +++ b/build/package-build-dockerfiles/centos7/Dockerfile @@ -48,7 +48,7 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ - /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 # Install Bash 4.4.12 // see https://dev.arvados.org/issues/15612 RUN cd /usr/local/src \ diff --git a/build/package-build-dockerfiles/debian10/Dockerfile b/build/package-build-dockerfiles/debian10/Dockerfile index c10d2dfeab..c0349aa798 100644 --- a/build/package-build-dockerfiles/debian10/Dockerfile +++ b/build/package-build-dockerfiles/debian10/Dockerfile @@ -53,7 +53,7 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ - /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ diff --git a/build/package-build-dockerfiles/debian11/Dockerfile b/build/package-build-dockerfiles/debian11/Dockerfile index c420ada61c..cccf8930d2 100644 --- a/build/package-build-dockerfiles/debian11/Dockerfile +++ b/build/package-build-dockerfiles/debian11/Dockerfile @@ -59,7 +59,7 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ - /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ diff --git a/build/package-build-dockerfiles/debian12/Dockerfile b/build/package-build-dockerfiles/debian12/Dockerfile new file mode 100644 index 0000000000..86d038b80b --- /dev/null +++ b/build/package-build-dockerfiles/debian12/Dockerfile @@ -0,0 +1,75 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +ARG HOSTTYPE +ARG BRANCH +ARG GOVERSION + +## dont use debian:12 here since the word 'bookworm' is used for rvm precompiled binaries +FROM debian:bookworm as build_x86_64 +# Install go +ONBUILD ARG GOVERSION +ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/ +ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/ +# Install nodejs and npm +ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/ +ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/ +# On x86, we want some cross-compilation support for arm64 +# Add gcc-aarch64-linux-gnu to compile go binaries for arm64 +ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu +# We also need libpam compiled for arm64 +ONBUILD RUN /usr/bin/dpkg --add-architecture arm64 +ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64 + +FROM debian:bookworm as build_aarch64 +# Install go +ONBUILD ARG GOVERSION +ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/ +ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/ +# Install nodejs and npm +ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/ +ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/ + +FROM build_${HOSTTYPE} +RUN echo HOSTTYPE ${HOSTTYPE} + +MAINTAINER Arvados Package Maintainers + +ENV DEBIAN_FRONTEND noninteractive + +SHELL ["/bin/bash", "-c"] +# Install dependencies. +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-setuptools python3-pip libcurl4-gnutls-dev curl git procps libattr1-dev libfuse-dev libgnutls28-dev libpq-dev unzip python3-venv python3-virtualenv python3-dev libpam-dev equivs + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) --disable-binary && \ + /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \ + echo "gem: --no-document" >> ~/.gemrc && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 + +RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) +# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ +ENV MAKE "make --jobs 8" + +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. +RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ + cd /tmp/arvados/services/api && \ + /usr/local/rvm/bin/rvm-exec default bundle install && \ + cd /tmp/arvados/apps/workbench && \ + /usr/local/rvm/bin/rvm-exec default bundle install && \ + cd /tmp/arvados && \ + go mod download + +ENV WORKSPACE /arvados +CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "debian12"] diff --git a/build/package-build-dockerfiles/rocky8/Dockerfile b/build/package-build-dockerfiles/rocky8/Dockerfile index ee999a8274..8bc9b0b79d 100644 --- a/build/package-build-dockerfiles/rocky8/Dockerfile +++ b/build/package-build-dockerfiles/rocky8/Dockerfile @@ -78,7 +78,7 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ - /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ diff --git a/build/package-build-dockerfiles/ubuntu1804/Dockerfile b/build/package-build-dockerfiles/ubuntu1804/Dockerfile index 27102f5876..304e2c925b 100644 --- a/build/package-build-dockerfiles/ubuntu1804/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu1804/Dockerfile @@ -52,7 +52,7 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ - /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ diff --git a/build/package-build-dockerfiles/ubuntu2004/Dockerfile b/build/package-build-dockerfiles/ubuntu2004/Dockerfile index ad1222c0fb..f85d5d6966 100644 --- a/build/package-build-dockerfiles/ubuntu2004/Dockerfile +++ b/build/package-build-dockerfiles/ubuntu2004/Dockerfile @@ -63,7 +63,7 @@ RUN gpg --import --no-tty /tmp/mpapis.asc && \ /usr/local/rvm/bin/rvm alias create default ruby-2.7 && \ echo "gem: --no-document" >> ~/.gemrc && \ /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ - /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.10.2 + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) # Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ diff --git a/build/package-build-dockerfiles/ubuntu2204/Dockerfile b/build/package-build-dockerfiles/ubuntu2204/Dockerfile new file mode 100644 index 0000000000..b6c0a14ea9 --- /dev/null +++ b/build/package-build-dockerfiles/ubuntu2204/Dockerfile @@ -0,0 +1,79 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +ARG HOSTTYPE +ARG BRANCH +ARG GOVERSION + +FROM ubuntu:jammy as build_x86_64 +# Install go +ONBUILD ARG GOVERSION +ONBUILD ADD generated/go${GOVERSION}.linux-amd64.tar.gz /usr/local/ +ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/ +# Install nodejs and npm +ONBUILD ADD generated/node-v10.23.1-linux-x64.tar.xz /usr/local/ +ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-x64/bin/* /usr/local/bin/ +# On x86, we want some cross-compilation support for arm64 +# Add gcc-aarch64-linux-gnu to compile go binaries for arm64 +ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y gcc-aarch64-linux-gnu +# We also need libpam compiled for arm64, and that requires some sources.list mangling +ONBUILD RUN /bin/sed -i 's/deb http/deb [ arch=amd64 ] http/' /etc/apt/sources.list +ONBUILD ADD ports.list /etc/apt/sources.list.d/ +ONBUILD RUN /usr/bin/dpkg --add-architecture arm64 +ONBUILD RUN /usr/bin/apt-get update && /usr/bin/apt-get install -o APT::Immediate-Configure=0 -q -y libpam0g-dev:arm64 libfuse-dev:arm64 + +FROM ubuntu:jammy as build_aarch64 +# Install go +ONBUILD ARG GOVERSION +ONBUILD ADD generated/go${GOVERSION}.linux-arm64.tar.gz /usr/local/ +ONBUILD RUN ln -s /usr/local/go/bin/go /usr/local/bin/ +# Install nodejs and npm +ONBUILD ADD generated/node-v10.23.1-linux-arm64.tar.xz /usr/local/ +ONBUILD RUN ln -s /usr/local/node-v10.23.1-linux-arm64/bin/* /usr/local/bin/ + +FROM build_${HOSTTYPE} + +LABEL org.opencontainers.image.authors="Arvados Package Maintainers " + +ENV DEBIAN_FRONTEND noninteractive + +SHELL ["/bin/bash", "-c"] +# Install dependencies. +RUN /usr/bin/apt-get update && /usr/bin/apt-get install -q -y python3 python3-pip libcurl4-gnutls-dev libgnutls28-dev curl git libattr1-dev libfuse-dev libpq-dev unzip tzdata python3-venv python3-dev libpam-dev shared-mime-info equivs + +# Install virtualenv +RUN /usr/bin/pip3 install 'virtualenv<20' + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \ + echo "gem: --no-document" >> ~/.gemrc && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 && \ + /usr/local/rvm/bin/rvm-exec default gem install fpm --version 1.15.1 + +RUN /usr/local/rvm/bin/rvm-exec default bundle config --global jobs $(let a=$(grep -c processor /proc/cpuinfo )-1; echo $a) +# Cf. https://build.betterup.com/one-weird-trick-that-will-speed-up-your-bundle-install/ +ENV MAKE "make --jobs 8" + +# Preseed the go module cache and the ruby gems, using the currently checked +# out branch of the source tree. This avoids potential compatibility issues +# between the version of Ruby and certain gems. +RUN git clone --depth 1 git://git.arvados.org/arvados.git /tmp/arvados && \ + cd /tmp/arvados && \ + if [[ -n "${BRANCH}" ]]; then git checkout ${BRANCH}; fi && \ + cd /tmp/arvados/services/api && \ + /usr/local/rvm/bin/rvm-exec default bundle install && \ + cd /tmp/arvados/apps/workbench && \ + /usr/local/rvm/bin/rvm-exec default bundle install && \ + cd /tmp/arvados && \ + go mod download + + +ENV WORKSPACE /arvados +CMD ["/usr/local/rvm/bin/rvm-exec", "default", "bash", "/jenkins/run-build-packages.sh", "--target", "ubuntu2204"] diff --git a/build/package-build-dockerfiles/ubuntu2204/ports.list b/build/package-build-dockerfiles/ubuntu2204/ports.list new file mode 100644 index 0000000000..a32f44e739 --- /dev/null +++ b/build/package-build-dockerfiles/ubuntu2204/ports.list @@ -0,0 +1,8 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe multiverse +deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe multiverse +deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted universe multiverse +deb [arch=arm64,armhf,ppc64el,s390x] http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/build/package-test-dockerfiles/Makefile b/build/package-test-dockerfiles/Makefile index 96a80055ac..f5287c53ea 100644 --- a/build/package-test-dockerfiles/Makefile +++ b/build/package-test-dockerfiles/Makefile @@ -2,32 +2,46 @@ # # SPDX-License-Identifier: AGPL-3.0 -all: centos7/generated debian10/generated debian11/generated rocky8/generated ubuntu1804/generated ubuntu2004/generated - +all: centos7/generated centos7/generated: common-generated-all test -d centos7/generated || mkdir centos7/generated cp -f -rlt centos7/generated common-generated/* +all: debian10/generated debian10/generated: common-generated-all test -d debian10/generated || mkdir debian10/generated cp -f -rlt debian10/generated common-generated/* +all: debian11/generated debian11/generated: common-generated-all test -d debian11/generated || mkdir debian11/generated cp -f -rlt debian11/generated common-generated/* +all: debian12/generated +debian12/generated: common-generated-all + test -d debian12/generated || mkdir debian12/generated + cp -f -rlt debian12/generated common-generated/* + +all: rocky8/generated rocky8/generated: common-generated-all test -d rocky8/generated || mkdir rocky8/generated cp -f -rlt rocky8/generated common-generated/* +all: ubuntu1804/generated ubuntu1804/generated: common-generated-all test -d ubuntu1804/generated || mkdir ubuntu1804/generated cp -f -rlt ubuntu1804/generated common-generated/* +all: ubuntu2004/generated ubuntu2004/generated: common-generated-all test -d ubuntu2004/generated || mkdir ubuntu2004/generated cp -f -rlt ubuntu2004/generated common-generated/* +all: ubuntu2204/generated +ubuntu2204/generated: common-generated-all + test -d ubuntu2204/generated || mkdir ubuntu2204/generated + cp -f -rlt ubuntu2204/generated common-generated/* + RVMKEY1=mpapis.asc RVMKEY2=pkuczynski.asc diff --git a/build/package-test-dockerfiles/debian12/Dockerfile b/build/package-test-dockerfiles/debian12/Dockerfile new file mode 100644 index 0000000000..4cdc41d73b --- /dev/null +++ b/build/package-test-dockerfiles/debian12/Dockerfile @@ -0,0 +1,28 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +FROM debian:bookworm +MAINTAINER Arvados Package Maintainers + +ENV DEBIAN_FRONTEND noninteractive + +# Install dependencies +RUN apt-get update && \ + apt-get -y install --no-install-recommends curl ca-certificates gpg procps gpg-agent + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) --disable-binary && \ + /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \ + echo "gem: --no-document" >> /etc/gemrc && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 + +# udev daemon can't start in a container, so don't try. +RUN mkdir -p /etc/udev/disabled + +RUN echo "deb file:///arvados/packages/debian12/ /" >>/etc/apt/sources.list diff --git a/build/package-test-dockerfiles/ubuntu2204/Dockerfile b/build/package-test-dockerfiles/ubuntu2204/Dockerfile new file mode 100644 index 0000000000..4926a6573f --- /dev/null +++ b/build/package-test-dockerfiles/ubuntu2204/Dockerfile @@ -0,0 +1,27 @@ +# Copyright (C) The Arvados Authors. All rights reserved. +# +# SPDX-License-Identifier: AGPL-3.0 + +FROM ubuntu:jammy +LABEL org.opencontainers.image.authors="Arvados Package Maintainers " + +ENV DEBIAN_FRONTEND noninteractive + +# Install dependencies +RUN apt-get update && \ + apt-get -y install --no-install-recommends curl ca-certificates gnupg2 + +# Install RVM +ADD generated/mpapis.asc /tmp/ +ADD generated/pkuczynski.asc /tmp/ +RUN gpg --import --no-tty /tmp/mpapis.asc && \ + gpg --import --no-tty /tmp/pkuczynski.asc && \ + curl -L https://get.rvm.io | bash -s stable && \ + /usr/local/rvm/bin/rvm install 3.2.2 -j $(grep -c processor /proc/cpuinfo) && \ + /usr/local/rvm/bin/rvm alias create default ruby-3.2.2 && \ + /usr/local/rvm/bin/rvm-exec default gem install bundler --version 2.2.19 + +# udev daemon can't start in a container, so don't try. +RUN mkdir -p /etc/udev/disabled + +RUN echo "deb [trusted=yes] file:///arvados/packages/ubuntu2204/ /" >>/etc/apt/sources.list diff --git a/build/package-testing/test-packages-debian12.sh b/build/package-testing/test-packages-debian12.sh new file mode 120000 index 0000000000..54ce94c357 --- /dev/null +++ b/build/package-testing/test-packages-debian12.sh @@ -0,0 +1 @@ +deb-common-test-packages.sh \ No newline at end of file diff --git a/build/package-testing/test-packages-ubuntu2204.sh b/build/package-testing/test-packages-ubuntu2204.sh new file mode 120000 index 0000000000..54ce94c357 --- /dev/null +++ b/build/package-testing/test-packages-ubuntu2204.sh @@ -0,0 +1 @@ +deb-common-test-packages.sh \ No newline at end of file diff --git a/build/run-library.sh b/build/run-library.sh index dd878d8475..aaeb931d34 100755 --- a/build/run-library.sh +++ b/build/run-library.sh @@ -442,8 +442,10 @@ test_package_presence() { declare -A dd dd[debian10]=buster dd[debian11]=bullseye + dd[debian12]=bookworm dd[ubuntu1804]=bionic dd[ubuntu2004]=focal + dd[ubuntu2204]=jammy D=${dd[$TARGET]} if [ ${pkgname:0:3} = "lib" ]; then repo_subdir=${pkgname:0:4} @@ -709,10 +711,17 @@ fpm_build_virtualenv_worker () { rm -rf dist/* - # Get the latest setuptools - if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'; then + # Get the latest setuptools. + # + # Note "pip3 install setuptools" fails on debian12 ("error: + # externally-managed-environment") even if that requirement is + # already satisfied, so we parse "pip3 list" output instead to check + # whether we need to do anything. + if [[ "$($pip list | grep -P -o '^setuptools\s+\K[0-9]+')" -ge 66 ]]; then + : # OK, already installed + elif ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools>=66'; then echo "Error, unable to upgrade setuptools with" - echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<45'" + echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools>=66'" exit 1 fi # filter a useless warning (when building the cwltest package) from the stderr output diff --git a/doc/_includes/_install_ruby_and_bundler.liquid b/doc/_includes/_install_ruby_and_bundler.liquid index f933082498..5d5bc9e9d7 100644 --- a/doc/_includes/_install_ruby_and_bundler.liquid +++ b/doc/_includes/_install_ruby_and_bundler.liquid @@ -4,23 +4,18 @@ Copyright (C) The Arvados Authors. All rights reserved. SPDX-License-Identifier: CC-BY-SA-3.0 {% endcomment %} -Ruby 2.6 or newer is required. +Ruby 2.7 or newer is required. * "Option 1: Install from packages":#packages * "Option 2: Install with RVM":#rvm -* "Option 3: Install from source":#fromsource h2(#packages). Option 1: Install from packages -{% include 'notebox_begin' %} -Future versions of Arvados may require a newer version of Ruby than is packaged with your OS. Using OS packages simplifies initial install, but may complicate upgrades that rely on a newer Ruby. If this is a concern, we recommend using "RVM":#rvm. -{% include 'notebox_end' %} - h3. Alma/CentOS/Red Hat/Rocky -The Ruby version shipped with version 7 of these distributions is too old. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer). +Version 7 of these distributions does not provide a new enough Ruby version. Use "RVM":#rvm to install Ruby 2.7 or newer. -Ruby 2.7 is available in a module with version 8 of these distributions. You can install it by running: +Version 8 of these distributions provides Ruby 2.7. You can install it by running: 
# dnf module enable ruby:2.7
@@ -29,7 +24,7 @@ Ruby 2.7 is available in a module with version 8 of these distributions. You can
 
 h3. Debian and Ubuntu
 
-Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install a newer version of Ruby (we recommend installing version 2.7 or newer).
+Debian 10 (buster) and Ubuntu 18.04 (bionic) ship with Ruby 2.5, which is too old for Arvados. Use "RVM":#rvm to install Ruby 2.7 or newer.
 
 Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or newer, which is sufficient for Arvados.
 
@@ -39,28 +34,42 @@ Debian 11 (bullseye) and Ubuntu 20.04 (focal) and later ship with Ruby 2.7 or ne
 
 h2(#rvm). Option 2: Install with RVM
 
+{% include 'notebox_begin_warning' %}
+We do not recommend using RVM unless the Ruby version provided by your OS distribution is older than 2.7.
+{% include 'notebox_end' %}
+
 h3. Install gpg and curl
 
-h4. Alma/CentOS/Red Hat/Rocky
+h4. CentOS/Red Hat 7
+
+
+yum install gpg curl which findutils procps
+

+
+{% comment %}
+To build ruby 3.2.2 on CentOS 7, add: "yum --enablerepo=powertools install libyaml-devel"
+{% endcomment %}
+
+h4. Alma/CentOS/Red Hat/Rocky 8+
 
 
-dnf install gpg curl which
+dnf install gpg curl which findutils procps
 

 
 h4. Debian and Ubuntu
 
 
-apt-get --no-install-recommends install gpg curl
+apt-get --no-install-recommends install gpg curl ca-certificates dirmngr procps
 

 
 h3. Install RVM, Ruby and Bundler
 
 
-
# gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
-\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7
+
gpg --keyserver pgp.mit.edu --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
+\curl -sSL https://get.rvm.io | bash -s stable --ruby=2.7.7
 

 
-This command installs the latest Ruby 2.7.x release, as well as the @gem@ and @bundle@ commands.
+This command installs the Ruby 2.7.7 release, as well as the @gem@ and @bundle@ commands.
 
 To use Ruby installed from RVM, load it in an open shell like this:
 
@@ -73,41 +82,3 @@ Alternately you can use @rvm-exec@ (the first parameter is the ruby version to u
 
 
rvm-exec default ruby -v
 

-
-h2(#fromsource). Option 3: Install from source
-
-Install prerequisites for Debian 10, Ubuntu 18.04 and Ubuntu 20.04:
-
-
-
sudo apt-get install \
-    bison build-essential gettext libcurl4 \
-    libcurl4-openssl-dev libpcre3-dev libreadline-dev \
-    libssl-dev libxslt1.1 zlib1g-dev
-

-
-Install prerequisites for Alma/CentOS/Red Hat/Rocky:
-
-
-
sudo dnf install \
-    libyaml-devel glibc-headers autoconf gcc-c++ glibc-devel \
-    patch readline-devel zlib-devel libffi-devel openssl-devel \
-    make automake libtool bison sqlite-devel tar
-

-
-Build and install Ruby:
-
-
-
mkdir -p ~/src
-cd ~/src
-curl -f https://cache.ruby-lang.org/pub/ruby/3.2/ruby-3.2.2.tar.gz | tar xz
-cd ruby-3.2.2
-./configure --disable-install-static-library --enable-shared --disable-install-doc
-make -j8
-sudo make install
-
-# Make sure the post install script can find the gem and ruby executables
-sudo ln -s /usr/local/bin/gem /usr/bin/gem
-sudo ln -s /usr/local/bin/ruby /usr/bin/ruby
-# Install bundler
-sudo -i gem install bundler --no-document
-

diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 739a08b5b3..48f8659dc7 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -32,6 +32,10 @@ h2(#main). development main
 
 "previous: Upgrading to 2.7.0":#v2_7_0
 
+h3. Remove Workbench1 packages after upgrading the salt installer
+
+If you installed a previous version of Arvados with the Salt installer, and you upgrade your installer to upgrade the cluster, you should uninstall the @arvados-workbench@ package from the workbench instance afterwards.
+
 h3. Remove Workbench1 packages and configuration
 
 The Workbench1 application has been removed from the Arvados distribution. We recommend the following follow-up steps.
diff --git a/doc/install/salt-multi-host.html.textile.liquid b/doc/install/salt-multi-host.html.textile.liquid
index eaffcf582c..a3cdd03300 100644
--- a/doc/install/salt-multi-host.html.textile.liquid
+++ b/doc/install/salt-multi-host.html.textile.liquid
@@ -233,10 +233,10 @@ The installer will set up the Arvados services on your machines.  Here is the de
 # KEEPSTORE nodes (at least 1 if using S3 as a Keep backend, else 2)
 ## arvados keepstore   (recommendend hostnames @keep0.${DOMAIN}@ and @keep1.${DOMAIN}@)
 # WORKBENCH node
-## arvados workbench   (recommendend hostname @workbench.${DOMAIN}@)
-## arvados workbench2  (recommendend hostname @workbench2.${DOMAIN}@)
-## arvados webshell    (recommendend hostname @webshell.${DOMAIN}@)
-## arvados websocket   (recommendend hostname @ws.${DOMAIN}@)
+## arvados legacy workbench URLs   (recommendend hostname @workbench.${DOMAIN}@)
+## arvados workbench2              (recommendend hostname @workbench2.${DOMAIN}@)
+## arvados webshell                (recommendend hostname @webshell.${DOMAIN}@)
+## arvados websocket               (recommendend hostname @ws.${DOMAIN}@)
 ## arvados cloud dispatcher
 ## arvados keepbalance
 ## arvados keepproxy   (recommendend hostname @keep.${DOMAIN}@)
@@ -284,7 +284,6 @@ BLOB_SIGNING_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 MANAGEMENT_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 SYSTEM_ROOT_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 ANONYMOUS_USER_TOKEN=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
-WORKBENCH_SECRET_KEY=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 DATABASE_PASSWORD=XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
 

 # Set @DATABASE_PASSWORD@ to a random string (unless you "already have a database":#ext-database then you should set it to that database's password)
@@ -409,13 +408,7 @@ The installer records log files for each deployment.
 
 Most service logs go to @/var/log/syslog@.
 
-The logs for Rails API server and for Workbench can be found in
-
-@/var/www/arvados-api/current/log/production.log@
-and
-@/var/www/arvados-workbench/current/log/production.log@
-
-on the appropriate instances.
+The logs for Rails API server can be found in @/var/www/arvados-api/current/log/production.log@ on the appropriate instance(s).
 
 Workbench 2 is a client-side Javascript application.  If you are having trouble loading Workbench 2, check the browser's developer console (this can be found in "Tools → Developer Tools").
 
diff --git a/doc/install/salt-single-host.html.textile.liquid b/doc/install/salt-single-host.html.textile.liquid
index 8ad79d86e8..92c1aa2645 100644
--- a/doc/install/salt-single-host.html.textile.liquid
+++ b/doc/install/salt-single-host.html.textile.liquid
@@ -48,7 +48,7 @@ Determine if you will use a single hostname, or multiple hostnames.
 
 If you are using multiple hostnames, determine the base domain for the cluster.  This will be referred to as @${DOMAIN}@.
 
-For example, if CLUSTER is @xarv1@ and DOMAIN is @example.com@, then @controller.${CLUSTER}.${DOMAIN}@" means @controller.xarv1.example.com@.
+For example, if CLUSTER is @xarv1@ and DOMAIN is @example.com@, then @controller.${CLUSTER}.${DOMAIN}@ means @controller.xarv1.example.com@.
 
 h3. Machine specification
 
@@ -190,13 +190,7 @@ The installer records log files for each deployment.
 
 Most service logs go to @/var/log/syslog@.
 
-The logs for Rails API server and for Workbench can be found in
-
-@/var/www/arvados-api/current/log/production.log@
-and
-@/var/www/arvados-workbench/current/log/production.log@
-
-on the appropriate instances.
+The logs for Rails API server can be found in @/var/www/arvados-api/current/log/production.log@ on the appropriate instance.
 
 Workbench 2 is a client-side Javascript application.  If you are having trouble loading Workbench 2, check the browser's developer console (this can be found in "Tools → Developer Tools").
 
diff --git a/sdk/java-v2/src/main/java/org/arvados/client/config/ExternalConfigProvider.java b/sdk/java-v2/src/main/java/org/arvados/client/config/ExternalConfigProvider.java
index d592b23ac3..e3d706ed0c 100644
--- a/sdk/java-v2/src/main/java/org/arvados/client/config/ExternalConfigProvider.java
+++ b/sdk/java-v2/src/main/java/org/arvados/client/config/ExternalConfigProvider.java
@@ -11,6 +11,10 @@ import java.io.File;
 
 public class ExternalConfigProvider implements ConfigProvider {
 
+    private static final int DEFAULT_CONNECTION_TIMEOUT = 60000;
+    private static final int DEFAULT_READ_TIMEOUT = 60000;
+    private static final int DEFAULT_WRITE_TIMEOUT = 60000;
+
     private boolean apiHostInsecure;
     private String keepWebHost;
     private int keepWebPort;
@@ -41,9 +45,9 @@ public class ExternalConfigProvider implements ConfigProvider {
         this.fileSplitDirectory = fileSplitDirectory;
         this.numberOfCopies = numberOfCopies;
         this.numberOfRetries = numberOfRetries;
-	this.connectTimeout = 60000;
-	this.readTimeout = 60000;
-	this.writeTimeout = 60000;
+	this.connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
+	this.readTimeout = DEFAULT_READ_TIMEOUT;
+	this.writeTimeout = DEFAULT_WRITE_TIMEOUT;
     }
 
     ExternalConfigProvider(boolean apiHostInsecure, String keepWebHost, int keepWebPort, String apiHost, int apiPort,
@@ -156,6 +160,9 @@ public class ExternalConfigProvider implements ConfigProvider {
         private File fileSplitDirectory;
         private int numberOfCopies;
         private int numberOfRetries;
+        private int connectTimeout = DEFAULT_CONNECTION_TIMEOUT;
+        private int readTimeout = DEFAULT_READ_TIMEOUT;
+        private int writeTimeout = DEFAULT_WRITE_TIMEOUT;
 
         ExternalConfigProviderBuilder() {
         }
@@ -215,8 +222,23 @@ public class ExternalConfigProvider implements ConfigProvider {
             return this;
         }
 
+        public ExternalConfigProvider.ExternalConfigProviderBuilder connectTimeout(int connectTimeout) {
+            this.connectTimeout = connectTimeout;
+            return this;
+        }
+
+        public ExternalConfigProvider.ExternalConfigProviderBuilder readTimeout(int readTimeout) {
+            this.readTimeout = readTimeout;
+            return this;
+        }
+
+        public ExternalConfigProvider.ExternalConfigProviderBuilder writeTimeout(int writeTimeout) {
+            this.writeTimeout = writeTimeout;
+            return this;
+        }
+
         public ExternalConfigProvider build() {
-            return new ExternalConfigProvider(apiHostInsecure, keepWebHost, keepWebPort, apiHost, apiPort, apiToken, apiProtocol, fileSplitSize, fileSplitDirectory, numberOfCopies, numberOfRetries);
+            return new ExternalConfigProvider(apiHostInsecure, keepWebHost, keepWebPort, apiHost, apiPort, apiToken, apiProtocol, fileSplitSize, fileSplitDirectory, numberOfCopies, numberOfRetries, connectTimeout, readTimeout, writeTimeout);
         }
 
     }
diff --git a/sdk/java-v2/src/test/java/org/arvados/client/facade/ArvadosFacadeIntegrationTest.java b/sdk/java-v2/src/test/java/org/arvados/client/facade/ArvadosFacadeIntegrationTest.java
index 07269f7e7d..05ba8d1b09 100644
--- a/sdk/java-v2/src/test/java/org/arvados/client/facade/ArvadosFacadeIntegrationTest.java
+++ b/sdk/java-v2/src/test/java/org/arvados/client/facade/ArvadosFacadeIntegrationTest.java
@@ -223,6 +223,9 @@ public class ArvadosFacadeIntegrationTest extends ArvadosClientIntegrationTest {
                 .fileSplitDirectory(CONFIG.getFileSplitDirectory())
                 .numberOfCopies(CONFIG.getNumberOfCopies())
                 .numberOfRetries(CONFIG.getNumberOfRetries())
+                .connectTimeout(CONFIG.getConnectTimeout())
+                .readTimeout(CONFIG.getReadTimeout())
+                .writeTimeout(CONFIG.getWriteTimeout())
                 .build();
     }
 
diff --git a/sdk/python/arvados/__init__.py b/sdk/python/arvados/__init__.py
index 21ca72c4bd..e90f381298 100644
--- a/sdk/python/arvados/__init__.py
+++ b/sdk/python/arvados/__init__.py
@@ -1,32 +1,30 @@
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
+"""Arvados Python SDK
+
+This module provides the entire Python SDK for Arvados. The most useful modules
+include:
+
+* arvados.api - After you `import arvados`, you can call `arvados.api.api` as
+  `arvados.api` to construct a client object.
+
+* arvados.collection - The `arvados.collection.Collection` class provides a
+  high-level interface to read and write collections. It coordinates sending
+  data to and from Keep, and synchronizing updates with the collection object.
+
+* arvados.util - Utility functions to use mostly in conjunction with the API
+  client object and the results it returns.
+
+Other submodules provide lower-level functionality.
+"""
 
-from __future__ import print_function
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import object
-import bz2
-import fcntl
-import hashlib
-import http.client
-import httplib2
-import json
 import logging as stdliblog
 import os
-import pprint
-import re
-import string
 import sys
-import time
 import types
-import zlib
 
-if sys.version_info >= (3, 0):
-    from collections import UserDict
-else:
-    from UserDict import UserDict
+from collections import UserDict
 
 from .api import api, api_from_config, http_cache
 from .collection import CollectionReader, CollectionWriter, ResumableCollectionWriter
diff --git a/sdk/python/arvados/api.py b/sdk/python/arvados/api.py
index ca9f17f866..8a17e42fcb 100644
--- a/sdk/python/arvados/api.py
+++ b/sdk/python/arvados/api.py
@@ -9,12 +9,7 @@ niceties such as caching, X-Request-Id header for tracking, and more. The main
 client constructors are `api` and `api_from_config`.
 """
 
-from __future__ import absolute_import
-from future import standard_library
-standard_library.install_aliases()
-from builtins import range
 import collections
-import http.client
 import httplib2
 import json
 import logging
@@ -28,6 +23,14 @@ import threading
 import time
 import types
 
+from typing import (
+    Any,
+    Dict,
+    List,
+    Mapping,
+    Optional,
+)
+
 import apiclient
 import apiclient.http
 from apiclient import discovery as apiclient_discovery
@@ -152,7 +155,7 @@ def _new_http_error(cls, *args, **kwargs):
         errors.ApiError, *args, **kwargs)
 apiclient_errors.HttpError.__new__ = staticmethod(_new_http_error)
 
-def http_cache(data_type):
+def http_cache(data_type: str) -> cache.SafeHTTPCache:
     """Set up an HTTP file cache
 
     This function constructs and returns an `arvados.cache.SafeHTTPCache`
@@ -177,18 +180,18 @@ def http_cache(data_type):
     return cache.SafeHTTPCache(str(path), max_age=60*60*24*2)
 
 def api_client(
-        version,
-        discoveryServiceUrl,
-        token,
+        version: str,
+        discoveryServiceUrl: str,
+        token: str,
         *,
-        cache=True,
-        http=None,
-        insecure=False,
-        num_retries=10,
-        request_id=None,
-        timeout=5*60,
-        **kwargs,
-):
+        cache: bool=True,
+        http: Optional[httplib2.Http]=None,
+        insecure: bool=False,
+        num_retries: int=10,
+        request_id: Optional[str]=None,
+        timeout: int=5*60,
+        **kwargs: Any,
+) -> apiclient_discovery.Resource:
     """Build an Arvados API client
 
     This function returns a `googleapiclient.discovery.Resource` object
@@ -232,7 +235,6 @@ def api_client(
 
     Additional keyword arguments will be passed directly to
     `googleapiclient.discovery.build`.
-
     """
     if http is None:
         http = httplib2.Http(
@@ -294,12 +296,12 @@ def api_client(
     return svc
 
 def normalize_api_kwargs(
-        version=None,
-        discoveryServiceUrl=None,
-        host=None,
-        token=None,
-        **kwargs,
-):
+        version: Optional[str]=None,
+        discoveryServiceUrl: Optional[str]=None,
+        host: Optional[str]=None,
+        token: Optional[str]=None,
+        **kwargs: Any,
+) -> Dict[str, Any]:
     """Validate kwargs from `api` and build kwargs for `api_client`
 
     This method takes high-level keyword arguments passed to the `api`
@@ -352,7 +354,11 @@ def normalize_api_kwargs(
         **kwargs,
     }
 
-def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
+def api_kwargs_from_config(
+        version: Optional[str]=None,
+        apiconfig: Optional[Mapping[str, str]]=None,
+        **kwargs: Any
+) -> Dict[str, Any]:
     """Build `api_client` keyword arguments from configuration
 
     This function accepts a mapping with Arvados configuration settings like
@@ -395,9 +401,18 @@ def api_kwargs_from_config(version=None, apiconfig=None, **kwargs):
         **kwargs,
     )
 
-def api(version=None, cache=True, host=None, token=None, insecure=False,
-        request_id=None, timeout=5*60, *,
-        discoveryServiceUrl=None, **kwargs):
+def api(
+        version: Optional[str]=None,
+        cache: bool=True,
+        host: Optional[str]=None,
+        token: Optional[str]=None,
+        insecure: bool=False,
+        request_id: Optional[str]=None,
+        timeout: int=5*60,
+        *,
+        discoveryServiceUrl: Optional[str]=None,
+        **kwargs: Any,
+) -> 'arvados.safeapi.ThreadSafeApiCache':
     """Dynamically build an Arvados API client
 
     This function provides a high-level "do what I mean" interface to build an
@@ -449,7 +464,11 @@ def api(version=None, cache=True, host=None, token=None, insecure=False,
     from .safeapi import ThreadSafeApiCache
     return ThreadSafeApiCache({}, {}, kwargs, version)
 
-def api_from_config(version=None, apiconfig=None, **kwargs):
+def api_from_config(
+        version: Optional[str]=None,
+        apiconfig: Optional[Mapping[str, str]]=None,
+        **kwargs: Any
+) -> 'arvados.safeapi.ThreadSafeApiCache':
     """Build an Arvados API client from a configuration mapping
 
     This function builds an Arvados API client from a mapping with user
diff --git a/sdk/python/arvados/retry.py b/sdk/python/arvados/retry.py
index ea8a6f65af..e9e574f5df 100644
--- a/sdk/python/arvados/retry.py
+++ b/sdk/python/arvados/retry.py
@@ -15,21 +15,28 @@ It also provides utility functions for common operations with `RetryLoop`:
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from builtins import range
-from builtins import object
 import functools
 import inspect
 import pycurl
 import time
 
 from collections import deque
+from typing import (
+    Callable,
+    Generic,
+    Optional,
+    TypeVar,
+)
 
 import arvados.errors
 
 _HTTP_SUCCESSES = set(range(200, 300))
 _HTTP_CAN_RETRY = set([408, 409, 423, 500, 502, 503, 504])
 
-class RetryLoop(object):
+CT = TypeVar('CT', bound=Callable)
+T = TypeVar('T')
+
+class RetryLoop(Generic[T]):
     """Coordinate limited retries of code.
 
     `RetryLoop` coordinates a loop that runs until it records a
@@ -53,12 +60,12 @@ class RetryLoop(object):
       it doesn't succeed.  This means the loop body could run at most
       `num_retries + 1` times.
 
-    * success_check: Callable --- This is a function that will be called
-      each time the loop saves a result.  The function should return `True`
-      if the result indicates the code succeeded, `False` if it represents a
-      permanent failure, and `None` if it represents a temporary failure.
-      If no function is provided, the loop will end after any result is
-      saved.
+    * success_check: Callable[[T], bool | None] --- This is a function that
+      will be called each time the loop saves a result.  The function should
+      return `True` if the result indicates the code succeeded, `False` if
+      it represents a permanent failure, and `None` if it represents a
+      temporary failure.  If no function is provided, the loop will end
+      after any result is saved.
 
     * backoff_start: float --- The number of seconds that must pass before
       the loop's second iteration.  Default 0, which disables all waiting.
@@ -73,9 +80,15 @@ class RetryLoop(object):
     * max_wait: float --- Maximum number of seconds to wait between
       retries. Default 60.
     """
-    def __init__(self, num_retries, success_check=lambda r: True,
-                 backoff_start=0, backoff_growth=2, save_results=1,
-                 max_wait=60):
+    def __init__(
+            self,
+            num_retries: int,
+            success_check: Callable[[T], Optional[bool]]=lambda r: True,
+            backoff_start: float=0,
+            backoff_growth: float=2,
+            save_results: int=1,
+            max_wait: float=60
+    ) -> None:
         self.tries_left = num_retries + 1
         self.check_result = success_check
         self.backoff_wait = backoff_start
@@ -87,11 +100,11 @@ class RetryLoop(object):
         self._running = None
         self._success = None
 
-    def __iter__(self):
+    def __iter__(self) -> 'RetryLoop':
         """Return an iterator of retries."""
         return self
 
-    def running(self):
+    def running(self) -> Optional[bool]:
         """Return whether this loop is running.
 
         Returns `None` if the loop has never run, `True` if it is still running,
@@ -100,7 +113,7 @@ class RetryLoop(object):
         """
         return self._running and (self._success is None)
 
-    def __next__(self):
+    def __next__(self) -> int:
         """Record a loop attempt.
 
         If the loop is still running, decrements the number of tries left and
@@ -121,7 +134,7 @@ class RetryLoop(object):
         self.tries_left -= 1
         return self.tries_left
 
-    def save_result(self, result):
+    def save_result(self, result: T) -> None:
         """Record a loop result.
 
         Save the given result, and end the loop if it indicates
@@ -133,8 +146,7 @@ class RetryLoop(object):
 
         Arguments:
 
-        * result: Any --- The result from this loop attempt to check and
-        save.
+        * result: T --- The result from this loop attempt to check and save.
         """
         if not self.running():
             raise arvados.errors.AssertionError(
@@ -143,7 +155,7 @@ class RetryLoop(object):
         self._success = self.check_result(result)
         self._attempts += 1
 
-    def success(self):
+    def success(self) -> Optional[bool]:
         """Return the loop's end state.
 
         Returns `True` if the loop recorded a successful result, `False` if it
@@ -151,7 +163,7 @@ class RetryLoop(object):
         """
         return self._success
 
-    def last_result(self):
+    def last_result(self) -> T:
         """Return the most recent result the loop saved.
 
         Raises `arvados.errors.AssertionError` if called before any result has
@@ -163,7 +175,7 @@ class RetryLoop(object):
             raise arvados.errors.AssertionError(
                 "queried loop results before any were recorded")
 
-    def attempts(self):
+    def attempts(self) -> int:
         """Return the number of results that have been saved.
 
         This count includes all kinds of results: success, permanent failure,
@@ -171,7 +183,7 @@ class RetryLoop(object):
         """
         return self._attempts
 
-    def attempts_str(self):
+    def attempts_str(self) -> str:
         """Return a human-friendly string counting saved results.
 
         This method returns '1 attempt' or 'N attempts', where the number
@@ -183,7 +195,7 @@ class RetryLoop(object):
             return '{} attempts'.format(self._attempts)
 
 
-def check_http_response_success(status_code):
+def check_http_response_success(status_code: int) -> Optional[bool]:
     """Convert a numeric HTTP status code to a loop control flag.
 
     This method takes a numeric HTTP status code and returns `True` if
@@ -213,7 +225,7 @@ def check_http_response_success(status_code):
     else:
         return None  # Get well soon, server.
 
-def retry_method(orig_func):
+def retry_method(orig_func: CT) -> CT:
     """Provide a default value for a method's num_retries argument.
 
     This is a decorator for instance and class methods that accept a
diff --git a/sdk/python/arvados/safeapi.py b/sdk/python/arvados/safeapi.py
index 3ecc72a950..56b92e8f08 100644
--- a/sdk/python/arvados/safeapi.py
+++ b/sdk/python/arvados/safeapi.py
@@ -7,12 +7,15 @@ This module provides `ThreadSafeApiCache`, a thread-safe, API-compatible
 Arvados API client.
 """
 
-from __future__ import absolute_import
-
-from builtins import object
 import sys
 import threading
 
+from typing import (
+    Any,
+    Mapping,
+    Optional,
+)
+
 from . import config
 from . import keep
 from . import util
@@ -30,27 +33,31 @@ class ThreadSafeApiCache(object):
 
     Arguments:
 
-    apiconfig: Mapping[str, str] | None
-    : A mapping with entries for `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`,
-      and optionally `ARVADOS_API_HOST_INSECURE`. If not provided, uses
+    * apiconfig: Mapping[str, str] | None --- A mapping with entries for
+      `ARVADOS_API_HOST`, `ARVADOS_API_TOKEN`, and optionally
+      `ARVADOS_API_HOST_INSECURE`. If not provided, uses
       `arvados.config.settings` to get these parameters from user
       configuration.  You can pass an empty mapping to build the client
       solely from `api_params`.
 
-    keep_params: Mapping[str, Any]
-    : Keyword arguments used to construct an associated
-      `arvados.keep.KeepClient`.
+    * keep_params: Mapping[str, Any] --- Keyword arguments used to construct
+      an associated `arvados.keep.KeepClient`.
 
-    api_params: Mapping[str, Any]
-    : Keyword arguments used to construct each thread's API client. These
-      have the same meaning as in the `arvados.api.api` function.
+    * api_params: Mapping[str, Any] --- Keyword arguments used to construct
+      each thread's API client. These have the same meaning as in the
+      `arvados.api.api` function.
 
-    version: str | None
-    : A string naming the version of the Arvados API to use. If not specified,
-      the code will log a warning and fall back to 'v1'.
+    * version: str | None --- A string naming the version of the Arvados API
+      to use. If not specified, the code will log a warning and fall back to
+      `'v1'`.
     """
-
-    def __init__(self, apiconfig=None, keep_params={}, api_params={}, version=None):
+    def __init__(
+            self,
+            apiconfig: Optional[Mapping[str, str]]=None,
+            keep_params: Optional[Mapping[str, Any]]={},
+            api_params: Optional[Mapping[str, Any]]={},
+            version: Optional[str]=None,
+    ) -> None:
         if apiconfig or apiconfig is None:
             self._api_kwargs = api.api_kwargs_from_config(version, apiconfig, **api_params)
         else:
@@ -60,7 +67,7 @@ class ThreadSafeApiCache(object):
         self.local = threading.local()
         self.keep = keep.KeepClient(api_client=self, **keep_params)
 
-    def localapi(self):
+    def localapi(self) -> 'googleapiclient.discovery.Resource':
         try:
             client = self.local.api
         except AttributeError:
@@ -69,6 +76,6 @@ class ThreadSafeApiCache(object):
             self.local.api = client
         return client
 
-    def __getattr__(self, name):
+    def __getattr__(self, name: str) -> Any:
         # Proxy nonexistent attributes to the thread-local API client.
         return getattr(self.localapi(), name)
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index 88adc8879b..050c67f68d 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -1,10 +1,13 @@
 # Copyright (C) The Arvados Authors. All rights reserved.
 #
 # SPDX-License-Identifier: Apache-2.0
+"""Arvados utilities
 
-from __future__ import division
-from builtins import range
+This module provides functions and constants that are useful across a variety
+of Arvados resource types, or extend the Arvados API client (see `arvados.api`).
+"""
 
+import errno
 import fcntl
 import functools
 import hashlib
@@ -13,30 +16,63 @@ import os
 import random
 import re
 import subprocess
-import errno
 import sys
 import warnings
 
 import arvados.errors
 
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    TypeVar,
+    Union,
+)
+
+T = TypeVar('T')
+
 HEX_RE = re.compile(r'^[0-9a-fA-F]+$')
+"""Regular expression to match a hexadecimal string (case-insensitive)"""
 CR_UNCOMMITTED = 'Uncommitted'
+"""Constant `state` value for uncommited container requests"""
 CR_COMMITTED = 'Committed'
+"""Constant `state` value for committed container requests"""
 CR_FINAL = 'Final'
+"""Constant `state` value for finalized container requests"""
 
 keep_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*')
+"""Regular expression to match any Keep block locator"""
 signed_locator_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+(\+\S+)*\+A\S+(\+\S+)*')
+"""Regular expression to match any Keep block locator with an access token hint"""
 portable_data_hash_pattern = re.compile(r'[0-9a-f]{32}\+[0-9]+')
+"""Regular expression to match any collection portable data hash"""
+manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
+"""Regular expression to match an Arvados collection manifest text"""
+keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a file path from a collection identified by portable data hash"""
+keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match a `keep:` URI with a collection identified by portable data hash"""
+
 uuid_pattern = re.compile(r'[a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}')
+"""Regular expression to match any Arvados object UUID"""
 collection_uuid_pattern = re.compile(r'[a-z0-9]{5}-4zz18-[a-z0-9]{15}')
+"""Regular expression to match any Arvados collection UUID"""
+container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
+"""Regular expression to match any Arvados container UUID"""
 group_uuid_pattern = re.compile(r'[a-z0-9]{5}-j7d0g-[a-z0-9]{15}')
-user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados group UUID"""
 link_uuid_pattern = re.compile(r'[a-z0-9]{5}-o0j2j-[a-z0-9]{15}')
+"""Regular expression to match any Arvados link UUID"""
+user_uuid_pattern = re.compile(r'[a-z0-9]{5}-tpzed-[a-z0-9]{15}')
+"""Regular expression to match any Arvados user UUID"""
 job_uuid_pattern = re.compile(r'[a-z0-9]{5}-8i9sb-[a-z0-9]{15}')
-container_uuid_pattern = re.compile(r'[a-z0-9]{5}-dz642-[a-z0-9]{15}')
-manifest_pattern = re.compile(r'((\S+)( +[a-f0-9]{32}(\+[0-9]+)(\+\S+)*)+( +[0-9]+:[0-9]+:\S+)+$)+', flags=re.MULTILINE)
-keep_file_locator_pattern = re.compile(r'([0-9a-f]{32}\+[0-9]+)/(.*)')
-keepuri_pattern = re.compile(r'keep:([0-9a-f]{32}\+[0-9]+)/(.*)')
+"""Regular expression to match any Arvados job UUID
+
+.. WARNING:: Deprecated
+   Arvados job resources are deprecated and will be removed in a future
+   release. Prefer the containers API instead.
+"""
 
 def _deprecated(version=None, preferred=None):
     """Mark a callable as deprecated in the SDK
@@ -47,12 +83,11 @@ def _deprecated(version=None, preferred=None):
     If the following arguments are given, they'll be included in the
     notices:
 
-    preferred: str | None
-    : The name of an alternative that users should use instead.
+    * preferred: str | None --- The name of an alternative that users should
+      use instead.
 
-    version: str | None
-    : The version of Arvados when the callable is scheduled to be
-      removed.
+    * version: str | None --- The version of Arvados when the callable is
+      scheduled to be removed.
     """
     if version is None:
         version = ''
@@ -91,6 +126,276 @@ def _deprecated(version=None, preferred=None):
         return deprecated_wrapper
     return deprecated_decorator
 
+def is_hex(s: str, *length_args: int) -> bool:
+    """Indicate whether a string is a hexadecimal number
+
+    This method returns true if all characters in the string are hexadecimal
+    digits. It is case-insensitive.
+
+    You can also pass optional length arguments to check that the string has
+    the expected number of digits. If you pass one integer, the string must
+    have that length exactly, otherwise the method returns False. If you
+    pass two integers, the string's length must fall within that minimum and
+    maximum (inclusive), otherwise the method returns False.
+
+    Arguments:
+
+    * s: str --- The string to check
+
+    * length_args: int --- Optional length limit(s) for the string to check
+    """
+    num_length_args = len(length_args)
+    if num_length_args > 2:
+        raise arvados.errors.ArgumentError(
+            "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
+    elif num_length_args == 2:
+        good_len = (length_args[0] <= len(s) <= length_args[1])
+    elif num_length_args == 1:
+        good_len = (len(s) == length_args[0])
+    else:
+        good_len = True
+    return bool(good_len and HEX_RE.match(s))
+
+def keyset_list_all(
+        fn: Callable[..., 'arvados.api_resources.ArvadosAPIRequest'],
+        order_key: str="created_at",
+        num_retries: int=0,
+        ascending: bool=True,
+        **kwargs: Any,
+) -> Iterator[Dict[str, Any]]:
+    """Iterate all Arvados resources from an API list call
+
+    This method takes a method that represents an Arvados API list call, and
+    iterates the objects returned by the API server. It can make multiple API
+    calls to retrieve and iterate all objects available from the API server.
+
+    Arguments:
+
+    * fn: Callable[..., arvados.api_resources.ArvadosAPIRequest] --- A
+      function that wraps an Arvados API method that returns a list of
+      objects. If you have an Arvados API client named `arv`, examples
+      include `arv.collections().list` and `arv.groups().contents`. Note
+      that you should pass the function *without* calling it.
+
+    * order_key: str --- The name of the primary object field that objects
+      should be sorted by. This name is used to build an `order` argument
+      for `fn`. Default `'created_at'`.
+
+    * num_retries: int --- This argument is passed through to
+      `arvados.api_resources.ArvadosAPIRequest.execute` for each API call. See
+      that method's docstring for details. Default 0 (meaning API calls will
+      use the `num_retries` value set when the Arvados API client was
+      constructed).
+
+    * ascending: bool --- Used to build an `order` argument for `fn`. If True,
+      all fields will be sorted in `'asc'` (ascending) order. Otherwise, all
+      fields will be sorted in `'desc'` (descending) order.
+
+    Additional keyword arguments will be passed directly to `fn` for each API
+    call. Note that this function sets `count`, `limit`, and `order` as part of
+    its work.
+    """
+    pagesize = 1000
+    kwargs["limit"] = pagesize
+    kwargs["count"] = 'none'
+    asc = "asc" if ascending else "desc"
+    kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
+    other_filters = kwargs.get("filters", [])
+
+    try:
+        select = set(kwargs['select'])
+    except KeyError:
+        pass
+    else:
+        select.add(order_key)
+        select.add('uuid')
+        kwargs['select'] = list(select)
+
+    nextpage = []
+    tot = 0
+    expect_full_page = True
+    seen_prevpage = set()
+    seen_thispage = set()
+    lastitem = None
+    prev_page_all_same_order_key = False
+
+    while True:
+        kwargs["filters"] = nextpage+other_filters
+        items = fn(**kwargs).execute(num_retries=num_retries)
+
+        if len(items["items"]) == 0:
+            if prev_page_all_same_order_key:
+                nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
+                prev_page_all_same_order_key = False
+                continue
+            else:
+                return
+
+        seen_prevpage = seen_thispage
+        seen_thispage = set()
+
+        for i in items["items"]:
+            # In cases where there's more than one record with the
+            # same order key, the result could include records we
+            # already saw in the last page.  Skip them.
+            if i["uuid"] in seen_prevpage:
+                continue
+            seen_thispage.add(i["uuid"])
+            yield i
+
+        firstitem = items["items"][0]
+        lastitem = items["items"][-1]
+
+        if firstitem[order_key] == lastitem[order_key]:
+            # Got a page where every item has the same order key.
+            # Switch to using uuid for paging.
+            nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
+            prev_page_all_same_order_key = True
+        else:
+            # Start from the last order key seen, but skip the last
+            # known uuid to avoid retrieving the same row twice.  If
+            # there are multiple rows with the same order key it is
+            # still likely we'll end up retrieving duplicate rows.
+            # That's handled by tracking the "seen" rows for each page
+            # so they can be skipped if they show up on the next page.
+            nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
+            prev_page_all_same_order_key = False
+
+def ca_certs_path(fallback: T=httplib2.CA_CERTS) -> Union[str, T]:
+    """Return the path of the best available source of CA certificates
+
+    This function checks various known paths that provide trusted CA
+    certificates, and returns the first one that exists. It checks:
+
+    * the path in the `SSL_CERT_FILE` environment variable (used by OpenSSL)
+    * `/etc/arvados/ca-certificates.crt`, respected by all Arvados software
+    * `/etc/ssl/certs/ca-certificates.crt`, the default store on Debian-based
+      distributions
+    * `/etc/pki/tls/certs/ca-bundle.crt`, the default store on Red Hat-based
+      distributions
+
+    If none of these paths exist, this function returns the value of `fallback`.
+
+    Arguments:
+
+    * fallback: T --- The value to return if none of the known paths exist.
+      The default value is the certificate store of Mozilla's trusted CAs
+      included with the Python [certifi][] package.
+
+    [certifi]: https://pypi.org/project/certifi/
+    """
+    for ca_certs_path in [
+        # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+        # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+        os.environ.get('SSL_CERT_FILE'),
+        # Arvados specific:
+        '/etc/arvados/ca-certificates.crt',
+        # Debian:
+        '/etc/ssl/certs/ca-certificates.crt',
+        # Red Hat:
+        '/etc/pki/tls/certs/ca-bundle.crt',
+        ]:
+        if ca_certs_path and os.path.exists(ca_certs_path):
+            return ca_certs_path
+    return fallback
+
+def new_request_id() -> str:
+    """Return a random request ID
+
+    This function generates and returns a random string suitable for use as a
+    `X-Request-Id` header value in the Arvados API.
+    """
+    rid = "req-"
+    # 2**104 > 36**20 > 2**103
+    n = random.getrandbits(104)
+    for _ in range(20):
+        c = n % 36
+        if c < 10:
+            rid += chr(c+ord('0'))
+        else:
+            rid += chr(c+ord('a')-10)
+        n = n // 36
+    return rid
+
+def get_config_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+    """Return an Arvados cluster's configuration, with caching
+
+    This function gets and returns the Arvados configuration from the API
+    server. It caches the result on the client object and reuses it on any
+    future calls.
+
+    Arguments:
+
+    * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+      object to use to retrieve and cache the Arvados cluster configuration.
+    """
+    if not svc._rootDesc.get('resources').get('configs', False):
+        # Old API server version, no config export endpoint
+        return {}
+    if not hasattr(svc, '_cached_config'):
+        svc._cached_config = svc.configs().get().execute()
+    return svc._cached_config
+
+def get_vocabulary_once(svc: 'arvados.api_resources.ArvadosAPIClient') -> Dict[str, Any]:
+    """Return an Arvados cluster's vocabulary, with caching
+
+    This function gets and returns the Arvados vocabulary from the API
+    server. It caches the result on the client object and reuses it on any
+    future calls.
+
+    .. HINT:: Low-level method
+       This is a relatively low-level wrapper around the Arvados API. Most
+       users will prefer to use `arvados.vocabulary.load_vocabulary`.
+
+    Arguments:
+
+    * svc: arvados.api_resources.ArvadosAPIClient --- The Arvados API client
+      object to use to retrieve and cache the Arvados cluster vocabulary.
+    """
+    if not svc._rootDesc.get('resources').get('vocabularies', False):
+        # Old API server version, no vocabulary export endpoint
+        return {}
+    if not hasattr(svc, '_cached_vocabulary'):
+        svc._cached_vocabulary = svc.vocabularies().get().execute()
+    return svc._cached_vocabulary
+
+def trim_name(collectionname: str) -> str:
+    """Limit the length of a name to fit within Arvados API limits
+
+    This function ensures that a string is short enough to use as an object
+    name in the Arvados API, leaving room for text that may be added by the
+    `ensure_unique_name` argument. If the source name is short enough, it is
+    returned unchanged. Otherwise, this function returns a string with excess
+    characters removed from the middle of the source string and replaced with
+    an ellipsis.
+
+    Arguments:
+
+    * collectionname: str --- The desired source name
+    """
+    max_name_len = 254 - 28
+
+    if len(collectionname) > max_name_len:
+        over = len(collectionname) - max_name_len
+        split = int(max_name_len/2)
+        collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
+
+    return collectionname
+
+@_deprecated('3.0', 'arvados.util.keyset_list_all')
+def list_all(fn, num_retries=0, **kwargs):
+    # Default limit to (effectively) api server's MAX_LIMIT
+    kwargs.setdefault('limit', sys.maxsize)
+    items = []
+    offset = 0
+    items_available = sys.maxsize
+    while len(items) < items_available:
+        c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
+        items += c['items']
+        items_available = c['items_available']
+        offset = c['offset'] + len(c['items'])
+    return items
+
 @_deprecated('3.0')
 def clear_tmpdir(path=None):
     """
@@ -428,174 +733,3 @@ def listdir_recursive(dirname, base=None, max_depth=None):
         else:
             allfiles += [ent_base]
     return allfiles
-
-def is_hex(s, *length_args):
-    """is_hex(s[, length[, max_length]]) -> boolean
-
-    Return True if s is a string of hexadecimal digits.
-    If one length argument is given, the string must contain exactly
-    that number of digits.
-    If two length arguments are given, the string must contain a number of
-    digits between those two lengths, inclusive.
-    Return False otherwise.
-    """
-    num_length_args = len(length_args)
-    if num_length_args > 2:
-        raise arvados.errors.ArgumentError(
-            "is_hex accepts up to 3 arguments ({} given)".format(1 + num_length_args))
-    elif num_length_args == 2:
-        good_len = (length_args[0] <= len(s) <= length_args[1])
-    elif num_length_args == 1:
-        good_len = (len(s) == length_args[0])
-    else:
-        good_len = True
-    return bool(good_len and HEX_RE.match(s))
-
-@_deprecated('3.0', 'arvados.util.keyset_list_all')
-def list_all(fn, num_retries=0, **kwargs):
-    # Default limit to (effectively) api server's MAX_LIMIT
-    kwargs.setdefault('limit', sys.maxsize)
-    items = []
-    offset = 0
-    items_available = sys.maxsize
-    while len(items) < items_available:
-        c = fn(offset=offset, **kwargs).execute(num_retries=num_retries)
-        items += c['items']
-        items_available = c['items_available']
-        offset = c['offset'] + len(c['items'])
-    return items
-
-def keyset_list_all(fn, order_key="created_at", num_retries=0, ascending=True, **kwargs):
-    pagesize = 1000
-    kwargs["limit"] = pagesize
-    kwargs["count"] = 'none'
-    asc = "asc" if ascending else "desc"
-    kwargs["order"] = ["%s %s" % (order_key, asc), "uuid %s" % asc]
-    other_filters = kwargs.get("filters", [])
-
-    try:
-        select = set(kwargs['select'])
-    except KeyError:
-        pass
-    else:
-        select.add(order_key)
-        select.add('uuid')
-        kwargs['select'] = list(select)
-
-    nextpage = []
-    tot = 0
-    expect_full_page = True
-    seen_prevpage = set()
-    seen_thispage = set()
-    lastitem = None
-    prev_page_all_same_order_key = False
-
-    while True:
-        kwargs["filters"] = nextpage+other_filters
-        items = fn(**kwargs).execute(num_retries=num_retries)
-
-        if len(items["items"]) == 0:
-            if prev_page_all_same_order_key:
-                nextpage = [[order_key, ">" if ascending else "<", lastitem[order_key]]]
-                prev_page_all_same_order_key = False
-                continue
-            else:
-                return
-
-        seen_prevpage = seen_thispage
-        seen_thispage = set()
-
-        for i in items["items"]:
-            # In cases where there's more than one record with the
-            # same order key, the result could include records we
-            # already saw in the last page.  Skip them.
-            if i["uuid"] in seen_prevpage:
-                continue
-            seen_thispage.add(i["uuid"])
-            yield i
-
-        firstitem = items["items"][0]
-        lastitem = items["items"][-1]
-
-        if firstitem[order_key] == lastitem[order_key]:
-            # Got a page where every item has the same order key.
-            # Switch to using uuid for paging.
-            nextpage = [[order_key, "=", lastitem[order_key]], ["uuid", ">" if ascending else "<", lastitem["uuid"]]]
-            prev_page_all_same_order_key = True
-        else:
-            # Start from the last order key seen, but skip the last
-            # known uuid to avoid retrieving the same row twice.  If
-            # there are multiple rows with the same order key it is
-            # still likely we'll end up retrieving duplicate rows.
-            # That's handled by tracking the "seen" rows for each page
-            # so they can be skipped if they show up on the next page.
-            nextpage = [[order_key, ">=" if ascending else "<=", lastitem[order_key]], ["uuid", "!=", lastitem["uuid"]]]
-            prev_page_all_same_order_key = False
-
-def ca_certs_path(fallback=httplib2.CA_CERTS):
-    """Return the path of the best available CA certs source.
-
-    This function searches for various distribution sources of CA
-    certificates, and returns the first it finds.  If it doesn't find any,
-    it returns the value of `fallback` (httplib2's CA certs by default).
-    """
-    for ca_certs_path in [
-        # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
-        # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
-        os.environ.get('SSL_CERT_FILE'),
-        # Arvados specific:
-        '/etc/arvados/ca-certificates.crt',
-        # Debian:
-        '/etc/ssl/certs/ca-certificates.crt',
-        # Red Hat:
-        '/etc/pki/tls/certs/ca-bundle.crt',
-        ]:
-        if ca_certs_path and os.path.exists(ca_certs_path):
-            return ca_certs_path
-    return fallback
-
-def new_request_id():
-    rid = "req-"
-    # 2**104 > 36**20 > 2**103
-    n = random.getrandbits(104)
-    for _ in range(20):
-        c = n % 36
-        if c < 10:
-            rid += chr(c+ord('0'))
-        else:
-            rid += chr(c+ord('a')-10)
-        n = n // 36
-    return rid
-
-def get_config_once(svc):
-    if not svc._rootDesc.get('resources').get('configs', False):
-        # Old API server version, no config export endpoint
-        return {}
-    if not hasattr(svc, '_cached_config'):
-        svc._cached_config = svc.configs().get().execute()
-    return svc._cached_config
-
-def get_vocabulary_once(svc):
-    if not svc._rootDesc.get('resources').get('vocabularies', False):
-        # Old API server version, no vocabulary export endpoint
-        return {}
-    if not hasattr(svc, '_cached_vocabulary'):
-        svc._cached_vocabulary = svc.vocabularies().get().execute()
-    return svc._cached_vocabulary
-
-def trim_name(collectionname):
-    """
-    trim_name takes a record name (collection name, project name, etc)
-    and trims it to fit the 255 character name limit, with additional
-    space for the timestamp added by ensure_unique_name, by removing
-    excess characters from the middle and inserting an ellipse
-    """
-
-    max_name_len = 254 - 28
-
-    if len(collectionname) > max_name_len:
-        over = len(collectionname) - max_name_len
-        split = int(max_name_len/2)
-        collectionname = collectionname[0:split] + "…" + collectionname[split+over:]
-
-    return collectionname
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 467a818cdc..787837b723 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -2,10 +2,6 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-from __future__ import print_function
-from __future__ import division
-from builtins import str
-from builtins import range
 import argparse
 import atexit
 import errno
@@ -18,7 +14,6 @@ import shlex
 import shutil
 import signal
 import socket
-import string
 import subprocess
 import sys
 import tempfile
@@ -26,10 +21,7 @@ import time
 import unittest
 import yaml
 
-try:
-    from urllib.parse import urlparse
-except ImportError:
-    from urlparse import urlparse
+from urllib.parse import urlparse
 
 MY_DIRNAME = os.path.dirname(os.path.realpath(__file__))
 if __name__ == '__main__' and os.path.exists(
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 81cdbfcd1c..9ee2cca410 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -24,6 +24,7 @@ class ArvadosModel < ApplicationRecord
   before_destroy :ensure_owner_uuid_is_permitted
   before_destroy :ensure_permission_to_destroy
   before_create :update_modified_by_fields
+  before_create :add_uuid_to_name, :if => Proc.new { @_add_uuid_to_name }
   before_update :maybe_update_modified_by_fields
   after_create :log_create
   after_update :log_update
@@ -471,8 +472,6 @@ class ArvadosModel < ApplicationRecord
   end
 
   def save_with_unique_name!
-    uuid_was = uuid
-    name_was = name
     max_retries = 2
     transaction do
       conn = ActiveRecord::Base.connection
@@ -503,24 +502,20 @@ class ArvadosModel < ApplicationRecord
 
         conn.exec_query 'ROLLBACK TO SAVEPOINT save_with_unique_name'
 
-        new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
-        if new_name == name
-          # If the database is fast enough to do two attempts in the
-          # same millisecond, we need to wait to ensure we try a
-          # different timestamp on each attempt.
-          sleep 0.002
-          new_name = "#{name_was} (#{db_current_time.utc.iso8601(3)})"
-        end
-
-        self[:name] = new_name
-        if uuid_was.nil? && !uuid.nil?
+        if uuid_was.nil?
+          # new record, the uuid caused a name collision (very
+          # unlikely but possible), so generate new uuid
           self[:uuid] = nil
           if self.is_a? Collection
-            # Reset so that is assigned to the new UUID
+            # Also needs to be reset
             self[:current_version_uuid] = nil
           end
+          # need to adjust the name after the uuid has been generated
+          add_uuid_to_make_unique_name
+        else
+          # existing record, just update the name directly.
+          add_uuid_to_name
         end
-
         retry
       end
     end
@@ -581,6 +576,26 @@ class ArvadosModel < ApplicationRecord
                           *ft[:param_out])
   end
 
+  @_add_uuid_to_name = false
+  def add_uuid_to_make_unique_name
+    @_add_uuid_to_name = true
+  end
+
+  def add_uuid_to_name
+    # Incorporate the random part of the UUID into the name.  This
+    # lets us prevent name collision but the part we add to the name
+    # is still somewhat meaningful (instead of generating a second
+    # random meaningless string).
+    #
+    # Because ArvadosModel is an abstract class and assign_uuid is
+    # part of HasUuid (which is included by the other concrete
+    # classes) the assign_uuid hook gets added (and run) after this
+    # one.  So we need to call assign_uuid here to make sure we have a
+    # uuid.
+    assign_uuid
+    self.name = "#{self.name[0..236]} (#{self.uuid[-15..-1]})"
+  end
+
   protected
 
   def self.deep_sort_hash(x)
diff --git a/services/api/lib/simulate_job_log.rb b/services/api/lib/simulate_job_log.rb
deleted file mode 100644
index abcf42eaa7..0000000000
--- a/services/api/lib/simulate_job_log.rb
+++ /dev/null
@@ -1,62 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'current_api_client'
-
-module SimulateJobLog
-  include CurrentApiClient
-  def replay(filename, multiplier = 1, simulated_job_uuid = nil)
-    raise "Environment must be development or test" unless [ 'test', 'development' ].include? ENV['RAILS_ENV']
-
-    multiplier = multiplier.to_f
-    multiplier = 1.0 if multiplier <= 0
-
-    actual_start_time = Time.now
-    log_start_time = nil
-
-    if simulated_job_uuid and (job = Job.where(uuid: simulated_job_uuid).first)
-      job_owner_uuid = job.owner_uuid
-    else
-      job_owner_uuid = system_user_uuid
-    end
-
-    act_as_system_user do
-      File.open(filename).each.with_index do |line, index|
-        cols = {}
-        cols[:timestamp], rest_of_line = line.split(' ', 2)
-        begin
-          cols[:timestamp] = Time.strptime( cols[:timestamp], "%Y-%m-%d_%H:%M:%S" )
-        rescue ArgumentError
-          if line =~ /^((?:Sun|Mon|Tue|Wed|Thu|Fri|Sat) (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{1,2} \d\d:\d\d:\d\d \d{4}) (.*)/
-            # Wed Nov 19 07:12:39 2014
-            cols[:timestamp] = Time.strptime( $1, "%a %b %d %H:%M:%S %Y" )
-            rest_of_line = $2
-          else
-              STDERR.puts "Ignoring log line because of unknown time format: #{line}"
-          end
-        end
-        cols[:job_uuid], cols[:pid], cols[:task], cols[:event_type], cols[:message] = rest_of_line.split(' ', 5)
-        # Override job uuid with a simulated one if specified
-        cols[:job_uuid] = simulated_job_uuid || cols[:job_uuid]
-        # determine when we want to simulate this log being created, based on the time multiplier
-        log_start_time = cols[:timestamp] if log_start_time.nil?
-        log_time = cols[:timestamp]
-        actual_elapsed_time = Time.now - actual_start_time
-        log_elapsed_time = log_time - log_start_time
-        modified_elapsed_time = log_elapsed_time / multiplier
-        pause_time = modified_elapsed_time - actual_elapsed_time
-        sleep pause_time if pause_time > 0
-
-        Log.new({
-          owner_uuid:  job_owner_uuid,
-          event_at:    Time.zone.local_to_utc(cols[:timestamp]),
-          object_uuid: cols[:job_uuid],
-          event_type:  cols[:event_type],
-          properties:  { 'text' => line }
-        }).save!
-      end
-    end
-
-  end
-end
diff --git a/services/api/lib/tasks/replay_job_log.rake b/services/api/lib/tasks/replay_job_log.rake
deleted file mode 100644
index 9c0f005275..0000000000
--- a/services/api/lib/tasks/replay_job_log.rake
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-require 'simulate_job_log'
-desc 'Simulate job logging from a file. Three arguments: log filename, time multipler (optional), simulated job uuid (optional). E.g. (use quotation marks if using spaces between args): rake "replay_job_log[log.txt, 2.0, qr1hi-8i9sb-nf3qk0xzwwz3lre]"'
-task :replay_job_log, [:filename, :multiplier, :uuid] => :environment do |t, args|
-  include SimulateJobLog
-  abort("No filename specified.") if args[:filename].blank?
-  replay( args[:filename], args[:multiplier].to_f, args[:uuid] )
-end
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 574cd366fc..43797035bc 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -409,7 +409,7 @@ EOS
         ensure_unique_name: true
       }
       assert_response :success
-      assert_match /^owned_by_active \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+      assert_match /^owned_by_active \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
     end
   end
 
@@ -1285,7 +1285,7 @@ EOS
     assert_equal false, json_response['is_trashed']
     assert_nil json_response['trash_at']
     assert_nil json_response['delete_at']
-    assert_match /^same name for trashed and persisted collections \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+    assert_match /^same name for trashed and persisted collections \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
   end
 
   test 'cannot show collection in trashed subproject' do
diff --git a/services/api/test/functional/arvados/v1/groups_controller_test.rb b/services/api/test/functional/arvados/v1/groups_controller_test.rb
index d8daa4bdd7..ee7f716c80 100644
--- a/services/api/test/functional/arvados/v1/groups_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/groups_controller_test.rb
@@ -474,7 +474,7 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
     assert_not_equal(new_project['uuid'],
                      groups(:aproject).uuid,
                      "create returned same uuid as existing project")
-    assert_match(/^A Project \(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z\)$/,
+    assert_match(/^A Project \(#{new_project['uuid'][-15..-1]}\)$/,
                  new_project['name'])
   end
 
@@ -800,7 +800,7 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
             ensure_unique_name: true
            }
       assert_response :success
-      assert_match /^trashed subproject 3 \(\d{4}-\d\d-\d\d.*?Z\)$/, json_response['name']
+      assert_match /^trashed subproject 3 \(#{json_response['uuid'][-15..-1]}\)$/, json_response['name']
     end
 
     test "move trashed subproject to new owner #{auth}" do
diff --git a/services/api/test/unit/container_request_test.rb b/services/api/test/unit/container_request_test.rb
index 98136aa53b..d25c08a579 100644
--- a/services/api/test/unit/container_request_test.rb
+++ b/services/api/test/unit/container_request_test.rb
@@ -1131,13 +1131,13 @@ class ContainerRequestTest < ActiveSupport::TestCase
     assert_equal ContainerRequest::Final, cr.state
     output_coll = Collection.find_by_uuid(cr.output_uuid)
     # Make sure the resulting output collection name include the original name
-    # plus the date
+    # plus the last 15 characters of uuid
     assert_not_equal output_name, output_coll.name,
                      "more than one collection with the same owner and name"
     assert output_coll.name.include?(output_name),
            "New name should include original name"
-    assert_match /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z/, output_coll.name,
-                 "New name should include ISO8601 date"
+    assert_match /#{output_coll.uuid[-15..-1]}/, output_coll.name,
+                 "New name should include last 15 characters of uuid"
   end
 
   [[0, :check_output_ttl_0],
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
index dc98c43ace..bd95c5a868 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/arvados.sls
@@ -54,7 +54,8 @@ arvados:
     #     - ruby-dev
     #     - zlib1g-dev
 
-  # config:
+  config:
+    check_command: /usr/bin/arvados-server config-check -strict=false -config
   #   file: /etc/arvados/config.yml
   #   user: root
   ## IMPORTANT!!!!!
@@ -105,7 +106,7 @@ arvados:
     ### KEYS
     secrets:
       blob_signing_key: __BLOB_SIGNING_KEY__
-      workbench_secret_key: __WORKBENCH_SECRET_KEY__
+      workbench_secret_key: "deprecated"
 
     Login:
       Test:
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
index de4c830906..82f1b91bb5 100644
--- a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_passenger.sls
@@ -58,35 +58,6 @@ nginx:
       events:
         worker_connections: {{ max_reqs * 3 + 1 }}
 
-  ### SNIPPETS
-  snippets:
-    # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
-    ssl_hardening_default.conf:
-      - ssl_session_timeout: 1d
-      - ssl_session_cache: 'shared:arvadosSSL:10m'
-      - ssl_session_tickets: 'off'
-
-      # intermediate configuration
-      - ssl_protocols: TLSv1.2 TLSv1.3
-      - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
-      - ssl_prefer_server_ciphers: 'off'
-
-      # HSTS (ngx_http_headers_module is required) (63072000 seconds)
-      - add_header: 'Strict-Transport-Security "max-age=63072000" always'
-
-      # OCSP stapling
-      - ssl_stapling: 'on'
-      - ssl_stapling_verify: 'on'
-
-      # verify chain of trust of OCSP response using Root CA and Intermediate certs
-      # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
-
-      # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
-      # - ssl_dhparam: /path/to/dhparam
-
-      # replace with the IP address of your resolver
-      # - resolver: 127.0.0.1
-
   ### SITES
   servers:
     managed:
diff --git a/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_snippets.sls b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_snippets.sls
new file mode 100644
index 0000000000..dfe17b57a1
--- /dev/null
+++ b/tools/salt-install/config_examples/multi_host/aws/pillars/nginx_snippets.sls
@@ -0,0 +1,35 @@
+---
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+### NGINX
+nginx:
+  ### SNIPPETS
+  snippets:
+    # Based on https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.4
+    ssl_hardening_default.conf:
+      - ssl_session_timeout: 1d
+      - ssl_session_cache: 'shared:arvadosSSL:10m'
+      - ssl_session_tickets: 'off'
+
+      # intermediate configuration
+      - ssl_protocols: TLSv1.2 TLSv1.3
+      - ssl_ciphers: ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+      - ssl_prefer_server_ciphers: 'off'
+
+      # HSTS (ngx_http_headers_module is required) (63072000 seconds)
+      - add_header: 'Strict-Transport-Security "max-age=63072000" always'
+
+      # OCSP stapling
+      - ssl_stapling: 'on'
+      - ssl_stapling_verify: 'on'
+
+      # verify chain of trust of OCSP response using Root CA and Intermediate certs
+      # - ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates
+
+      # curl https://ssl-config.mozilla.org/ffdhe2048.txt > /path/to/dhparam
+      # - ssl_dhparam: /path/to/dhparam
+
+      # replace with the IP address of your resolver
+      # - resolver: 127.0.0.1
diff --git a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
index 5883f19241..275c2c78ab 100644
--- a/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/multiple_hostnames/pillars/arvados.sls
@@ -53,7 +53,8 @@ arvados:
     #     - ruby-dev
     #     - zlib1g-dev
 
-  # config:
+  config:
+    check_command: /usr/bin/arvados-server config-check -strict=false -config
   #   file: /etc/arvados/config.yml
   #   user: root
   ## IMPORTANT!!!!!
@@ -106,7 +107,7 @@ arvados:
     ### KEYS
     secrets:
       blob_signing_key: __BLOB_SIGNING_KEY__
-      workbench_secret_key: __WORKBENCH_SECRET_KEY__
+      workbench_secret_key: "deprecated"
 
     Login:
       Test:
diff --git a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
index e85b709c2c..f83984b01a 100644
--- a/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
+++ b/tools/salt-install/config_examples/single_host/single_hostname/pillars/arvados.sls
@@ -53,7 +53,8 @@ arvados:
     #     - ruby-dev
     #     - zlib1g-dev
 
-  # config:
+  config:
+    check_command: /usr/bin/arvados-server config-check -strict=false -config
   #   file: /etc/arvados/config.yml
   #   user: root
   ## IMPORTANT!!!!!
@@ -106,7 +107,7 @@ arvados:
     ### KEYS
     secrets:
       blob_signing_key: __BLOB_SIGNING_KEY__
-      workbench_secret_key: __WORKBENCH_SECRET_KEY__
+      workbench_secret_key: "deprecated"
 
     Login:
       Test:
diff --git a/tools/salt-install/installer.sh b/tools/salt-install/installer.sh
index 27feffa2d2..439293c296 100755
--- a/tools/salt-install/installer.sh
+++ b/tools/salt-install/installer.sh
@@ -281,7 +281,7 @@ terraform-destroy)
   ;;
 
 generate-tokens)
-  for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN WORKBENCH_SECRET_KEY DATABASE_PASSWORD; do
+  for i in BLOB_SIGNING_KEY MANAGEMENT_TOKEN SYSTEM_ROOT_TOKEN ANONYMOUS_USER_TOKEN DATABASE_PASSWORD; do
     echo ${i}=$(
       tr -dc A-Za-z0-9 > ${STATES_TOP}
   echo "    - logrotate" >> ${STATES_TOP}
   echo "    - docker.software" >> ${STATES_TOP}
-  echo "    - arvados" >> ${STATES_TOP}
+  echo "    - arvados.repo" >> ${STATES_TOP}
+  echo "    - arvados.config" >> ${STATES_TOP}
+  echo "    - arvados.ruby" >> ${STATES_TOP}
+  echo "    - arvados.api" >> ${STATES_TOP}
+  echo "    - arvados.controller" >> ${STATES_TOP}
+  echo "    - arvados.keepstore" >> ${STATES_TOP}
+  echo "    - arvados.websocket" >> ${STATES_TOP}
+  echo "    - arvados.keepweb" >> ${STATES_TOP}
+  echo "    - arvados.workbench2" >> ${STATES_TOP}
+  echo "    - arvados.keepproxy" >> ${STATES_TOP}
+  echo "    - arvados.shell" >> ${STATES_TOP}
+  echo "    - arvados.dispatcher" >> ${STATES_TOP}
   echo "    - extra.shell_sudo_passwordless" >> ${STATES_TOP}
   echo "    - extra.shell_cron_add_login_sync" >> ${STATES_TOP}
   echo "    - extra.passenger_rvm" >> ${STATES_TOP}
@@ -753,6 +763,7 @@ else
         for SVC in grafana prometheus; do
           grep -q "nginx_${SVC}_configuration" ${PILLARS_TOP} || echo "    - nginx_${SVC}_configuration" >> ${PILLARS_TOP}
         done
+        grep -q "nginx_snippets" ${PILLARS_TOP} || echo "    - nginx_snippets" >> ${PILLARS_TOP}
         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           grep -q "letsencrypt"     ${PILLARS_TOP} || echo "    - letsencrypt" >> ${PILLARS_TOP}
           for SVC in grafana prometheus; do
@@ -845,6 +856,7 @@ else
         grep -q "aws_credentials" ${PILLARS_TOP}          || echo "    - aws_credentials" >> ${PILLARS_TOP}
         grep -q "postgresql" ${PILLARS_TOP}               || echo "    - postgresql" >> ${PILLARS_TOP}
         grep -q "nginx_passenger" ${PILLARS_TOP}          || echo "    - nginx_passenger" >> ${PILLARS_TOP}
+        grep -q "nginx_snippets" ${PILLARS_TOP}           || echo "    - nginx_snippets" >> ${PILLARS_TOP}
         grep -q "nginx_api_configuration" ${PILLARS_TOP} || echo "    - nginx_api_configuration" >> ${PILLARS_TOP}
         grep -q "nginx_controller_configuration" ${PILLARS_TOP} || echo "    - nginx_controller_configuration" >> ${PILLARS_TOP}
 
@@ -875,17 +887,7 @@ else
       ;;
       "websocket" | "workbench" | "workbench2" | "webshell" | "keepweb" | "keepproxy")
         ### States ###
-        if [ "${R}" = "workbench" ]; then
-          grep -q "    - logrotate" ${STATES_TOP} || echo "    - logrotate" >> ${STATES_TOP}
-          NGINX_INSTALL_SOURCE="install_from_phusionpassenger"
-          if grep -q "    - nginx$" ${STATES_TOP}; then
-            sed -i s/"^    - nginx.*$"/"    - nginx.passenger"/g ${STATES_TOP}
-          else
-            echo "    - nginx.passenger" >> ${STATES_TOP}
-          fi
-        else
-          grep -q "\- nginx$" ${STATES_TOP} || echo "    - nginx" >> ${STATES_TOP}
-        fi
+        grep -q "\- nginx$" ${STATES_TOP} || echo "    - nginx" >> ${STATES_TOP}
 
         if [ "${SSL_MODE}" = "lets-encrypt" ]; then
           if [ "x${USE_LETSENCRYPT_ROUTE53:-}" = "xyes" ]; then
@@ -907,16 +909,14 @@ else
         fi
 
         # webshell role is just a nginx vhost, so it has no state
-        if [ "${R}" != "webshell" ]; then
+        # workbench role is deprecated since 2.7.0
+        if [[ "${R}" != "webshell" && "${R}" != "workbench" ]]; then
           grep -q "arvados.${R}" ${STATES_TOP} || echo "    - arvados.${R}" >> ${STATES_TOP}
         fi
 
         ### Pillars ###
-        if [ "${R}" = "workbench" ]; then
-          grep -q "logrotate_wb1" ${PILLARS_TOP} || echo "    - logrotate_wb1" >> ${PILLARS_TOP}
-        fi
-        grep -q "nginx_passenger" ${PILLARS_TOP}          || echo "    - nginx_passenger" >> ${PILLARS_TOP}
         grep -q "nginx_${R}_configuration" ${PILLARS_TOP} || echo "    - nginx_${R}_configuration" >> ${PILLARS_TOP}
+        grep -q "nginx_snippets" ${PILLARS_TOP} || echo "    - nginx_snippets" >> ${PILLARS_TOP}
         # Special case for keepweb
         if [ ${R} = "keepweb" ]; then
           grep -q "nginx_download_configuration" ${PILLARS_TOP} || echo "    - nginx_download_configuration" >> ${PILLARS_TOP}