From: Ward Vandewege <ward@curii.com>
Date: Fri, 17 Jul 2020 13:14:59 +0000 (-0400)
Subject: Merge branch 'patch-1' of https://github.com/mr-c/arvados into mr-c-patch-1
X-Git-Tag: 2.1.0~163^2
X-Git-Url: https://git.arvados.org/arvados.git/commitdiff_plain/7407f41105f8000bb3908d41a31daaf3a30d9440?hp=4d1fa0d132e32d02cb87c1c6e3e4c2742bdec467

Merge branch 'patch-1' of https://github.com/mr-c/arvados into mr-c-patch-1

No issue #

Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward@curii.com>
---

diff --git a/.licenseignore b/.licenseignore
index ad80dc3f4b..81f6b7181d 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -79,4 +79,6 @@ lib/dispatchcloud/test/sshkey_*
 *.asc
 sdk/java-v2/build.gradle
 sdk/java-v2/settings.gradle
-sdk/cwl/tests/wf/feddemo
\ No newline at end of file
+sdk/cwl/tests/wf/feddemo
+go.mod
+go.sum
diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index 2af9c8b16f..cb4e7ab9e3 100644
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -214,7 +214,7 @@ GEM
       multi_json (~> 1.0)
       websocket-driver (>= 0.2.0)
     public_suffix (4.0.3)
-    rack (2.0.7)
+    rack (2.2.3)
     rack-mini-profiler (1.0.2)
       rack (>= 1.2.0)
     rack-test (0.6.3)
@@ -315,7 +315,7 @@ GEM
       json (>= 1.8.0)
     websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.3)
+    websocket-extensions (0.1.5)
     xpath (2.1.0)
       nokogiri (~> 1.3)
 
@@ -375,4 +375,4 @@ DEPENDENCIES
   uglifier (~> 2.0)
 
 BUNDLED WITH
-   1.11
+   1.16.6
diff --git a/apps/workbench/app/views/layouts/application.html.erb b/apps/workbench/app/views/layouts/application.html.erb
index 4fc7da9949..c0f01da283 100644
--- a/apps/workbench/app/views/layouts/application.html.erb
+++ b/apps/workbench/app/views/layouts/application.html.erb
@@ -20,7 +20,7 @@ SPDX-License-Identifier: AGPL-3.0 %>
   <meta name="author" content="">
   <% if current_user %>
     <% content_for :js do %>
-      window.defaultSession = <%=raw({baseURL: Rails.configuration.Services.Controller.ExternalURL.to_s.gsub(/\/?$/,'/'), token: Thread.current[:arvados_api_token], user: current_user}.to_json)%>
+      window.defaultSession = <%=raw({baseURL: Rails.configuration.Services.Controller.ExternalURL.to_s.sub(/\/*$/,'/'), token: Thread.current[:arvados_api_token], user: current_user}.to_json)%>
     <% end %>
   <% end %>
   <% if current_user and $arvados_api_client.discovery[:websocketUrl] %>
@@ -49,7 +49,7 @@ SPDX-License-Identifier: AGPL-3.0 %>
   <%= yield :head %>
   <%= javascript_tag do %>
     angular.module('Arvados').value('arvadosApiToken', '<%=Thread.current[:arvados_api_token]%>');
-    angular.module('Arvados').value('arvadosDiscoveryUri', '<%= Rails.configuration.Services.Controller.ExternalURL.to_s + '/discovery/v1/apis/arvados/v1/rest' %>');
+    angular.module('Arvados').value('arvadosDiscoveryUri', '<%= Rails.configuration.Services.Controller.ExternalURL.to_s.sub(/\/*$/,'/') + 'discovery/v1/apis/arvados/v1/rest' %>');
   <%= yield :js %>
   <% end %>
   <style>
diff --git a/apps/workbench/test/integration/anonymous_access_test.rb b/apps/workbench/test/integration/anonymous_access_test.rb
index 0842635f60..cbbe28a6f3 100644
--- a/apps/workbench/test/integration/anonymous_access_test.rb
+++ b/apps/workbench/test/integration/anonymous_access_test.rb
@@ -117,6 +117,7 @@ class AnonymousAccessTest < ActionDispatch::IntegrationTest
   end
 
   test 'view file' do
+    need_selenium "phantomjs does not follow redirects reliably, maybe https://github.com/ariya/phantomjs/issues/10389"
     magic = rand(2**512).to_s 36
     owner = api_fixture('groups')['anonymously_accessible_project']['uuid']
     col = upload_data_and_get_collection(magic, 'admin', "Hello\\040world.txt", owner)
diff --git a/apps/workbench/test/integration/collections_test.rb b/apps/workbench/test/integration/collections_test.rb
index 87d3d678d1..e7b27fff86 100644
--- a/apps/workbench/test/integration/collections_test.rb
+++ b/apps/workbench/test/integration/collections_test.rb
@@ -53,6 +53,8 @@ class CollectionsTest < ActionDispatch::IntegrationTest
   end
 
   test "can download an entire collection with a reader token" do
+    need_selenium "phantomjs does not follow redirects reliably, maybe https://github.com/ariya/phantomjs/issues/10389"
+
     token = api_token('active')
     data = "foo\nfile\n"
     datablock = `echo -n #{data.shellescape} | ARVADOS_API_TOKEN=#{token.shellescape} arv-put --no-progress --raw -`.strip
@@ -68,24 +70,16 @@ class CollectionsTest < ActionDispatch::IntegrationTest
     token = api_fixture('api_client_authorizations')['active_all_collections']['api_token']
     url_head = "/collections/download/#{uuid}/#{token}/"
     visit url_head
+    assert_text "You can download individual files listed below"
     # It seems that Capybara can't inspect tags outside the body, so this is
     # a very blunt approach.
     assert_no_match(/<\s*meta[^>]+\bnofollow\b/i, page.html,
                     "wget prohibited from recursing the collection page")
     # Look at all the links that wget would recurse through using our
     # recommended options, and check that it's exactly the file list.
-    hrefs = page.all('a').map do |anchor|
-      link = anchor[:href] || ''
-      if link.start_with? url_head
-        link[url_head.size .. -1]
-      elsif link.start_with? '/'
-        nil
-      else
-        link
-      end
-    end
-    assert_equal(['./foo'], hrefs.compact.sort,
-                 "download page did provide strictly file links")
+    hrefs = []
+    page.html.scan(/href="(.*?)"/) { |m| hrefs << m[0] }
+    assert_equal(['./foo'], hrefs, "download page did provide strictly file links")
     click_link "foo"
     assert_text "foo\nfile\n"
   end
diff --git a/apps/workbench/test/integration/projects_test.rb b/apps/workbench/test/integration/projects_test.rb
index 17ab5e4661..7a5103007f 100644
--- a/apps/workbench/test/integration/projects_test.rb
+++ b/apps/workbench/test/integration/projects_test.rb
@@ -132,7 +132,7 @@ class ProjectsTest < ActionDispatch::IntegrationTest
     show_object_using('active', 'groups', 'aproject', 'A Project')
     click_on "Sharing"
     click_on "Share with groups"
-    good_uuid = api_fixture("groups")["private"]["uuid"]
+    good_uuid = api_fixture("groups")["future_project_viewing_group"]["uuid"]
     assert(page.has_selector?(".selectable[data-object-uuid=\"#{good_uuid}\"]"),
            "'share with groups' listing missing owned user group")
     bad_uuid = api_fixture("groups")["asubproject"]["uuid"]
diff --git a/apps/workbench/test/integration/user_settings_menu_test.rb b/apps/workbench/test/integration/user_settings_menu_test.rb
index 5f2886c7a3..99076bbaf7 100644
--- a/apps/workbench/test/integration/user_settings_menu_test.rb
+++ b/apps/workbench/test/integration/user_settings_menu_test.rb
@@ -188,7 +188,7 @@ class UserSettingsMenuTest < ActionDispatch::IntegrationTest
     end
     assert_text ":active/workbenchtest.git"
     assert_match /git@git.*:active\/workbenchtest.git/, page.text
-    assert_match /#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}\/active\/workbenchtest.git/, page.text
+    assert_match /#{Rails.configuration.Services.GitHTTP.ExternalURL.to_s}active\/workbenchtest.git/, page.text
   end
 
   [
diff --git a/build/build-dev-docker-jobs-image.sh b/build/build-dev-docker-jobs-image.sh
index a3d439be6f..7da8089837 100755
--- a/build/build-dev-docker-jobs-image.sh
+++ b/build/build-dev-docker-jobs-image.sh
@@ -69,15 +69,7 @@ fi
 
 . build/run-library.sh
 
-python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
-
-if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
-    cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
-fi
+calculate_python_sdk_cwl_package_versions
 
 set -x
 docker build --no-cache --build-arg sdk=$sdk --build-arg runner=$runner --build-arg salad=$salad --build-arg cwltool=$cwltool --build-arg pythoncmd=$py --build-arg pipcmd=$pipcmd -f "$WORKSPACE/sdk/dev-jobs.dockerfile" -t arvados/jobs:$cwl_runner_version "$WORKSPACE/sdk"
diff --git a/build/package-testing/test-package-python3-arvados-cwl-runner.sh b/build/package-testing/test-package-python3-arvados-cwl-runner.sh
new file mode 100755
index 0000000000..99327c016a
--- /dev/null
+++ b/build/package-testing/test-package-python3-arvados-cwl-runner.sh
@@ -0,0 +1,8 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arvados-cwl-runner --version
diff --git a/build/package-testing/test-package-python3-arvados-python-client.sh b/build/package-testing/test-package-python3-arvados-python-client.sh
new file mode 100755
index 0000000000..d4e66a27b9
--- /dev/null
+++ b/build/package-testing/test-package-python3-arvados-python-client.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arv-put --version
+
+/usr/share/python3/dist/python3-arvados-python-client/bin/python3 << EOF
+import arvados
+print("Successfully imported arvados")
+EOF
diff --git a/build/package-testing/test-package-python3-python-arvados-fuse.sh b/build/package-testing/test-package-python3-python-arvados-fuse.sh
new file mode 120000
index 0000000000..3b9232c5fa
--- /dev/null
+++ b/build/package-testing/test-package-python3-python-arvados-fuse.sh
@@ -0,0 +1 @@
+test-package-python27-python-arvados-fuse.sh
\ No newline at end of file
diff --git a/build/package-testing/test-package-rh-python36-python-arvados-python-client.sh b/build/package-testing/test-package-rh-python36-python-arvados-python-client.sh
new file mode 100755
index 0000000000..1a69256560
--- /dev/null
+++ b/build/package-testing/test-package-rh-python36-python-arvados-python-client.sh
@@ -0,0 +1,13 @@
+#!/bin/sh
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+set -e
+
+arv-put --version
+
+/usr/share/python3/dist/rh-python36-python-arvados-python-client/bin/python3 << EOF
+import arvados
+print("Successfully imported arvados")
+EOF
diff --git a/build/rails-package-scripts/arvados-sso-server.sh b/build/rails-package-scripts/arvados-sso-server.sh
index fff582bb18..e88da0d3a6 100644
--- a/build/rails-package-scripts/arvados-sso-server.sh
+++ b/build/rails-package-scripts/arvados-sso-server.sh
@@ -8,6 +8,6 @@
 PACKAGE_NAME=arvados-sso-server
 INSTALL_PATH=/var/www/arvados-sso
 CONFIG_PATH=/etc/arvados/sso
-DOC_URL="http://doc.arvados.org/install/install-sso.html#configure"
+DOC_URL="https://doc.arvados.org/v2.0/install/install-sso.html#configure"
 RAILSPKG_DATABASE_LOAD_TASK=db:schema:load
 RAILSPKG_SUPPORTS_CONFIG_CHECK=0
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index 842975adb0..ec8357701d 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -139,15 +139,7 @@ if [[ -z "$ARVADOS_BUILDING_VERSION" ]] && ! [[ -z "$version_tag" ]]; then
 	ARVADOS_BUILDING_ITERATION="1"
 fi
 
-python_sdk_ts=$(cd sdk/python && timestamp_from_git)
-cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
-
-python_sdk_version=$(cd sdk/python && nohash_version_from_git 0.1)
-cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git 1.0)
-
-if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
-    cwl_runner_version=$(cd sdk/python && nohash_version_from_git 1.0)
-fi
+calculate_python_sdk_cwl_package_versions
 
 echo cwl_runner_version $cwl_runner_version python_sdk_version $python_sdk_version
 
diff --git a/build/run-build-packages-one-target.sh b/build/run-build-packages-one-target.sh
index f476a9691c..f8816dbe48 100755
--- a/build/run-build-packages-one-target.sh
+++ b/build/run-build-packages-one-target.sh
@@ -143,6 +143,22 @@ if [[ -n "$test_packages" ]]; then
   fi
 
   if [[ -n "$(find $WORKSPACE/packages/$TARGET -name '*.deb')" ]] ; then
+    set +e
+    /usr/bin/which dpkg-scanpackages >/dev/null
+    if [[ "$?" != "0" ]]; then
+      echo >&2
+      echo >&2 "Error: please install dpkg-dev. E.g. sudo apt-get install dpkg-dev"
+      echo >&2
+      exit 1
+    fi
+    /usr/bin/which apt-ftparchive >/dev/null
+    if [[ "$?" != "0" ]]; then
+      echo >&2
+      echo >&2 "Error: please install apt-utils. E.g. sudo apt-get install apt-utils"
+      echo >&2
+      exit 1
+    fi
+    set -e
     (cd $WORKSPACE/packages/$TARGET
       dpkg-scanpackages .  2> >(grep -v 'warning' 1>&2) | tee Packages | gzip -c > Packages.gz
       apt-ftparchive -o APT::FTPArchive::Release::Origin=Arvados release . > Release
@@ -192,6 +208,8 @@ if test -z "$packages" ; then
         keepstore
         keep-web
         libarvados-perl
+        libpam-arvados
+        libpam-arvados-go
         python-arvados-fuse
         python-arvados-python-client
         python-arvados-cwl-runner"
diff --git a/build/run-build-packages-python-and-ruby.sh b/build/run-build-packages-python-and-ruby.sh
index f9b61179ca..ba44218c4e 100755
--- a/build/run-build-packages-python-and-ruby.sh
+++ b/build/run-build-packages-python-and-ruby.sh
@@ -21,6 +21,10 @@ Options:
 --upload
     If the build and test steps are successful, upload the python
     packages to pypi and the gems to rubygems (default: false)
+--ruby <true|false>
+    Build ruby gems (default: true)
+--python <true|false>
+    Build python packages (default: true)
 
 WORKSPACE=path         Path to the Arvados source tree to build packages from
 
@@ -65,10 +69,12 @@ python_wrapper() {
 
 TARGET=
 UPLOAD=0
+RUBY=1
+PYTHON=1
 DEBUG=${ARVADOS_DEBUG:-0}
 
 PARSEDOPTS=$(getopt --name "$0" --longoptions \
-    help,debug,upload,target: \
+    help,debug,ruby:,python:,upload,target: \
     -- "" "$@")
 if [ $? -ne 0 ]; then
     exit 1
@@ -85,6 +91,22 @@ while [ $# -gt 0 ]; do
         --target)
             TARGET="$2"; shift
             ;;
+        --ruby)
+            RUBY="$2"; shift
+            if [ "$RUBY" != "true" ] && [ "$RUBY" != "1" ]; then
+              RUBY=0
+            else
+              RUBY=1
+            fi
+            ;;
+        --python)
+            PYTHON="$2"; shift
+            if [ "$PYTHON" != "true" ] && [ "$PYTHON" != "1" ]; then
+              PYTHON=0
+            else
+              PYTHON=1
+            fi
+            ;;
         --upload)
             UPLOAD=1
             ;;
@@ -129,6 +151,11 @@ fi
 debug_echo "$0 is running from $RUN_BUILD_PACKAGES_PATH"
 debug_echo "Workspace is $WORKSPACE"
 
+if [ $RUBY -eq 0 ] && [ $PYTHON -eq 0 ]; then
+  echo "Nothing to do!"
+  exit 0
+fi
+
 if [[ -f /etc/profile.d/rvm.sh ]]; then
     source /etc/profile.d/rvm.sh
     GEM="rvm-exec default gem"
@@ -150,60 +177,69 @@ umask 0022
 
 debug_echo "umask is" `umask`
 
-gem_wrapper arvados "$WORKSPACE/sdk/ruby"
-gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
-gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
-
 GEM_BUILD_FAILURES=0
-if [ ${#failures[@]} -ne 0 ]; then
-  GEM_BUILD_FAILURES=${#failures[@]}
+if [ $RUBY -eq 1 ]; then
+  debug_echo "Building Ruby gems"
+  gem_wrapper arvados "$WORKSPACE/sdk/ruby"
+  gem_wrapper arvados-cli "$WORKSPACE/sdk/cli"
+  gem_wrapper arvados-login-sync "$WORKSPACE/services/login-sync"
+  if [ ${#failures[@]} -ne 0 ]; then
+    GEM_BUILD_FAILURES=${#failures[@]}
+  fi
 fi
 
-python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
-python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
-python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
-python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
-python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
-
 PYTHON_BUILD_FAILURES=0
-if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
-  PYTHON_BUILD_FAILURES=${#failures[@]} - $GEM_BUILD_FAILURES
+if [ $PYTHON -eq 1 ]; then
+  debug_echo "Building Python packages"
+  python_wrapper arvados-python-client "$WORKSPACE/sdk/python"
+  python_wrapper arvados-pam "$WORKSPACE/sdk/pam"
+  python_wrapper arvados-cwl-runner "$WORKSPACE/sdk/cwl"
+  python_wrapper arvados_fuse "$WORKSPACE/services/fuse"
+  python_wrapper arvados-node-manager "$WORKSPACE/services/nodemanager"
+
+  if [ $((${#failures[@]} - $GEM_BUILD_FAILURES)) -ne 0 ]; then
+    PYTHON_BUILD_FAILURES=$((${#failures[@]} - $GEM_BUILD_FAILURES))
+  fi
 fi
 
-if [[ "$UPLOAD" != 0 ]]; then
+if [ $UPLOAD -ne 0 ]; then
+  echo "Uploading"
 
-  if [[ $DEBUG > 0 ]]; then
+  if [ $DEBUG > 0 ]; then
     EXTRA_UPLOAD_FLAGS=" --verbose"
   else
     EXTRA_UPLOAD_FLAGS=""
   fi
 
-  if [[ ! -e "$WORKSPACE/packages" ]]; then
+  if [ ! -e "$WORKSPACE/packages" ]; then
     mkdir -p "$WORKSPACE/packages"
   fi
 
-  title "Start upload python packages"
-  timer_reset
-
-  if [ "$PYTHON_BUILD_FAILURES" -eq 0 ]; then
-    /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
-  else
-    echo "Skipping python packages upload, there were errors building the packages"
+  if [ $PYTHON -eq 1 ]; then
+    title "Start upload python packages"
+    timer_reset
+
+    if [ $PYTHON_BUILD_FAILURES -eq 0 ]; then
+      /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE python
+    else
+      echo "Skipping python packages upload, there were errors building the packages"
+    fi
+    checkexit $? "upload python packages"
+    title "End of upload python packages (`timer`)"
   fi
-  checkexit $? "upload python packages"
-  title "End of upload python packages (`timer`)"
 
-  title "Start upload ruby gems"
-  timer_reset
-
-  if [ "$GEM_BUILD_FAILURES" -eq 0 ]; then
-    /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
-  else
-    echo "Skipping ruby gem upload, there were errors building the packages"
+  if [ $RUBY -eq 1 ]; then
+    title "Start upload ruby gems"
+    timer_reset
+
+    if [ $GEM_BUILD_FAILURES -eq 0 ]; then
+      /usr/local/arvados-dev/jenkins/run_upload_packages.py $EXTRA_UPLOAD_FLAGS --workspace $WORKSPACE gems
+    else
+      echo "Skipping ruby gem upload, there were errors building the packages"
+    fi
+    checkexit $? "upload ruby gems"
+    title "End of upload ruby gems (`timer`)"
   fi
-  checkexit $? "upload ruby gems"
-  title "End of upload ruby gems (`timer`)"
-
 fi
 
 exit_cleanly
diff --git a/build/run-build-packages.sh b/build/run-build-packages.sh
index 3ba1dcc05e..5aa0b7e6f8 100755
--- a/build/run-build-packages.sh
+++ b/build/run-build-packages.sh
@@ -3,8 +3,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
-. `dirname "$(readlink -f "$0")"`/run-library.sh
-. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh
+. `dirname "$(readlink -f "$0")"`/run-library.sh || exit 1
+. `dirname "$(readlink -f "$0")"`/libcloud-pin.sh || exit 1
 
 read -rd "\000" helpmessage <<EOF
 $(basename $0): Build Arvados packages
@@ -223,7 +223,7 @@ if [[ -z "$ONLY_BUILD" ]] || [[ "libarvados-perl" = "$ONLY_BUILD" ]] ; then
 
     perl Makefile.PL INSTALL_BASE=install >"$STDOUT_IF_DEBUG" && \
         make install INSTALLDIRS=perl >"$STDOUT_IF_DEBUG" && \
-        fpm_build install/lib/=/usr/share libarvados-perl \
+        fpm_build "$WORKSPACE/sdk/perl" install/lib/=/usr/share libarvados-perl \
         dir "$(version_from_git)" install/man/=/usr/share/man \
         "$WORKSPACE/apache-2.0.txt=/usr/share/doc/libarvados-perl/apache-2.0.txt" && \
         mv --no-clobber libarvados-perl*.$FORMAT "$WORKSPACE/packages/$TARGET/"
@@ -271,7 +271,7 @@ debug_echo -e "\nPython packages\n"
       cd "$SRC_BUILD_DIR"
       PKG_VERSION=$(version_from_git)
       cd $WORKSPACE/packages/$TARGET
-      fpm_build $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
+      fpm_build "$WORKSPACE" $SRC_BUILD_DIR/=/usr/local/arvados/src arvados-src 'dir' "$PKG_VERSION" "--exclude=usr/local/arvados/src/.git" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=The Arvados source code" "--architecture=all"
 
       rm -rf "$SRC_BUILD_DIR"
     fi
@@ -318,6 +318,8 @@ package_go_binary tools/keep-rsync keep-rsync \
     "Copy all data from one set of Keep servers to another"
 package_go_binary tools/keep-exercise keep-exercise \
     "Performance testing tool for Arvados Keep"
+package_go_so lib/pam pam_arvados.so libpam-arvados-go \
+    "Arvados PAM authentication module"
 
 # The Python SDK - Should be built first because it's needed by others
 fpm_build_virtualenv "arvados-python-client" "sdk/python"
@@ -334,6 +336,9 @@ fpm_build_virtualenv "libpam-arvados" "sdk/pam"
 # The FUSE driver
 fpm_build_virtualenv "arvados-fuse" "services/fuse"
 
+# The FUSE driver - Python3 package
+fpm_build_virtualenv "arvados-fuse" "services/fuse" "python3"
+
 # The node manager
 fpm_build_virtualenv "arvados-node-manager" "services/nodemanager"
 
diff --git a/build/run-library.sh b/build/run-library.sh
index fd37f632b0..3e6c9f8584 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -48,7 +48,6 @@ version_from_git() {
     # Output the version being built, or if we're building a
     # dev/prerelease, output a version number based on the git log for
     # the given $subdir.
-    local minorversion="$1"; shift # unused
     local subdir="$1"; shift
     if [[ -n "$ARVADOS_BUILDING_VERSION" ]]; then
         echo "$ARVADOS_BUILDING_VERSION"
@@ -66,7 +65,7 @@ nohash_version_from_git() {
         echo "$ARVADOS_BUILDING_VERSION"
         return
     fi
-    version_from_git $1 | cut -d. -f1-4
+    version_from_git | cut -d. -f1-4
 }
 
 timestamp_from_git() {
@@ -74,6 +73,18 @@ timestamp_from_git() {
     format_last_commit_here "%ct" "$subdir"
 }
 
+calculate_python_sdk_cwl_package_versions() {
+  python_sdk_ts=$(cd sdk/python && timestamp_from_git)
+  cwl_runner_ts=$(cd sdk/cwl && timestamp_from_git)
+
+  python_sdk_version=$(cd sdk/python && nohash_version_from_git)
+  cwl_runner_version=$(cd sdk/cwl && nohash_version_from_git)
+
+  if [[ $python_sdk_ts -gt $cwl_runner_ts ]]; then
+    cwl_runner_version=$python_sdk_version
+  fi
+}
+
 handle_python_package () {
   # This function assumes the current working directory is the python package directory
   if [ -n "$(find dist -name "*-$(nohash_version_from_git).tar.gz" -print -quit)" ]; then
@@ -127,7 +138,7 @@ calculate_go_package_version() {
       cd "$WORKSPACE"
       ts="$(timestamp_from_git "$dir")"
       if [[ "$ts" -gt "$timestamp" ]]; then
-          version=$(version_from_git "" "$dir")
+          version=$(version_from_git "$dir")
           timestamp="$ts"
       fi
   done
@@ -135,7 +146,7 @@ calculate_go_package_version() {
   __returnvar="$version"
 }
 
-# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision"
+# Usage: package_go_binary services/foo arvados-foo "Compute foo to arbitrary precision" [apache-2.0.txt]
 package_go_binary() {
     local src_path="$1"; shift
     local prog="$1"; shift
@@ -174,7 +185,37 @@ package_go_binary() {
     fi
     switches+=("$WORKSPACE/${license_file}=/usr/share/doc/$prog/${license_file}")
 
-    fpm_build "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+    fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${basename}=/usr/bin/${prog}" "${prog}" dir "${go_package_version}" "--url=https://arvados.org" "--license=GNU Affero General Public License, version 3.0" "--description=${description}" "${switches[@]}"
+}
+
+# Usage: package_go_so lib/foo arvados_foo.so arvados-foo "Arvados foo library"
+package_go_so() {
+    local src_path="$1"; shift
+    local sofile="$1"; shift
+    local pkg="$1"; shift
+    local description="$1"; shift
+
+    debug_echo "package_go_so $src_path as $pkg"
+
+    calculate_go_package_version go_package_version $src_path
+    cd $WORKSPACE/packages/$TARGET
+    test_package_presence $pkg $go_package_version go || return 1
+    cd $WORKSPACE/$src_path
+    go build -buildmode=c-shared -o ${GOPATH}/bin/${sofile}
+    cd $WORKSPACE/packages/$TARGET
+    local -a fpmargs=(
+        "--url=https://arvados.org"
+        "--license=Apache License, Version 2.0"
+        "--description=${description}"
+        "$WORKSPACE/apache-2.0.txt=/usr/share/doc/$pkg/apache-2.0.txt"
+    )
+    if [[ -e "$WORKSPACE/$src_path/pam-configs-arvados" ]]; then
+        fpmargs+=("$WORKSPACE/$src_path/pam-configs-arvados=/usr/share/pam-configs/arvados-go")
+    fi
+    if [[ -e "$WORKSPACE/$src_path/README" ]]; then
+        fpmargs+=("$WORKSPACE/$src_path/README=/usr/share/doc/$pkg/README")
+    fi
+    fpm_build "${WORKSPACE}/${src_path}" "$GOPATH/bin/${sofile}=/usr/lib/${sofile}" "${pkg}" dir "${go_package_version}" "${fpmargs[@]}"
 }
 
 default_iteration() {
@@ -406,7 +447,7 @@ handle_rails_package() {
     for exclude in ${exclude_list[@]}; do
         switches+=(-x "$exclude_root/$exclude")
     done
-    fpm_build "${pos_args[@]}" "${switches[@]}" \
+    fpm_build "${srcdir}" "${pos_args[@]}" "${switches[@]}" \
               -x "$exclude_root/vendor/cache-*" \
               -x "$exclude_root/vendor/bundle" "$@" "$license_arg"
     rm -rf "$scripts_dir"
@@ -465,7 +506,7 @@ fpm_build_virtualenv () {
   fi
 
   # arvados-python-client sdist should always be built, to be available
-  # for other dependant packages.
+  # for other dependent packages.
   if [[ -n "$ONLY_BUILD" ]] && [[ "arvados-python-client" != "$PKG" ]] && [[ "$PYTHON_PKG" != "$ONLY_BUILD" ]] && [[ "$PKG" != "$ONLY_BUILD" ]]; then
     return 0
   fi
@@ -563,7 +604,7 @@ fpm_build_virtualenv () {
   cd build/usr/share/$python/dist/$PYTHON_PKG/
 
   # Replace the shebang lines in all python scripts, and handle the activate
-  # scripts too This is a functional replacement of the 237 line
+  # scripts too. This is a functional replacement of the 237 line
   # virtualenv_tools.py script that doesn't work in python3 without serious
   # patching, minus the parts we don't need (modifying pyc files, etc).
   for binfile in `ls bin/`; do
@@ -613,7 +654,7 @@ fpm_build_virtualenv () {
   # 12271 - As FPM-generated packages don't include scripts by default, the
   # packages cleanup on upgrade depends on files being listed on the %files
   # section in the generated SPEC files. To remove DIRECTORIES, they need to
-  # be listed in that sectiontoo, so we need to add this parameter to properly
+  # be listed in that section too, so we need to add this parameter to properly
   # remove lingering dirs. But this only works for python2: if used on
   # python33, it includes dirs like /opt/rh/python33 that belong to
   # other packages.
@@ -621,7 +662,7 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=('--rpm-auto-add-directories')
   fi
 
-  if [[ "$PKG" == "arvados-python-client" ]]; then
+  if [[ "$PKG" == "arvados-python-client" ]] || [[ "$PKG" == "arvados-fuse" ]]; then
     if [[ "$python" == "python2.7" ]]; then
       COMMAND_ARR+=('--conflicts' "$PYTHON3_PKG_PREFIX-$PKG")
     else
@@ -638,7 +679,8 @@ fpm_build_virtualenv () {
   COMMAND_ARR+=('-n' "$PYTHON_PKG")
   COMMAND_ARR+=('-C' "build")
 
-  if [[ -e "$WORKSPACE/$PKG_DIR/$PKG.service" ]]; then
+  systemd_unit="$WORKSPACE/$PKG_DIR/$PKG.service"
+  if [[ -e "${systemd_unit}" ]]; then
     COMMAND_ARR+=('--after-install' "${WORKSPACE}/build/go-python-package-scripts/postinst")
     COMMAND_ARR+=('--before-remove' "${WORKSPACE}/build/go-python-package-scripts/prerm")
   fi
@@ -671,6 +713,12 @@ fpm_build_virtualenv () {
     COMMAND_ARR+=('--depends' "$i")
   done
 
+  # make sure the systemd service file ends up in the right place
+  # used by arvados-docker-cleaner and arvados-node-manager
+  if [[ -e "${systemd_unit}" ]]; then
+    COMMAND_ARR+=("usr/share/$python/dist/$PKG/share/doc/$PKG/$PKG.service=/lib/systemd/system/$PKG.service")
+  fi
+
   COMMAND_ARR+=("${fpm_args[@]}")
 
   # Make sure to install all our package binaries in /usr/bin.
@@ -718,6 +766,9 @@ fpm_build_virtualenv () {
 
 # Build packages for everything
 fpm_build () {
+  # Source dir where fpm-info.sh (if any) will be found.
+  SRC_DIR=$1
+  shift
   # The package source.  Depending on the source type, this can be a
   # path, or the name of the package in an upstream repository (e.g.,
   # pip).
@@ -794,17 +845,15 @@ fpm_build () {
   declare -a build_depends=()
   declare -a fpm_depends=()
   declare -a fpm_exclude=()
-  declare -a fpm_dirs=(
-      # source dir part of 'dir' package ("/source=/dest" => "/source"):
-      "${PACKAGE%%=/*}")
-  for pkgdir in "${fpm_dirs[@]}"; do
-      fpminfo="$pkgdir/fpm-info.sh"
-      if [[ -e "$fpminfo" ]]; then
-          debug_echo "Loading fpm overrides from $fpminfo"
-          source "$fpminfo"
-          break
-      fi
-  done
+  if [[ ! -d "$SRC_DIR" ]]; then
+      echo >&2 "BUG: looking in wrong dir for fpm-info.sh: $pkgdir"
+      exit 1
+  fi
+  fpminfo="${SRC_DIR}/fpm-info.sh"
+  if [[ -e "$fpminfo" ]]; then
+      debug_echo "Loading fpm overrides from $fpminfo"
+      source "$fpminfo"
+  fi
   for pkg in "${build_depends[@]}"; do
       if [[ $TARGET =~ debian|ubuntu ]]; then
           pkg_deb=$(ls "$WORKSPACE/packages/$TARGET/$pkg_"*.deb | sort -rg | awk 'NR==1')
diff --git a/build/run-tests.sh b/build/run-tests.sh
index 0212d1bc0e..ff6ead0fac 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1205,6 +1205,7 @@ help_interactive() {
     echo "== Interactive commands:"
     echo "TARGET                 (short for 'test DIR')"
     echo "test TARGET"
+    echo "10 test TARGET         (run test 10 times)"
     echo "test TARGET:py3        (test with python3)"
     echo "test TARGET -check.vv  (pass arguments to test)"
     echo "install TARGET"
@@ -1265,6 +1266,10 @@ else
     while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do
         history -s "$nextcmd"
         history -w
+        count=1
+        if [[ "${nextcmd}" =~ ^[0-9] ]]; then
+          read count nextcmd <<<"${nextcmd}"
+        fi
         read verb target opts <<<"${nextcmd}"
         target="${target%/}"
         target="${target/\/:/:}"
@@ -1284,11 +1289,14 @@ else
                         ${verb}_${target}
                         ;;
                     *)
-			argstarget=${target%:py3}
+                        argstarget=${target%:py3}
                         testargs["$argstarget"]="${opts}"
                         tt="${testfuncargs[${target}]}"
                         tt="${tt:-$target}"
-                        do_$verb $tt
+                        while [ $count -gt 0 ]; do
+                          do_$verb $tt
+                          let "count=count-1"
+                        done
                         ;;
                 esac
                 ;;
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index 887bc62bb3..bcc3dda09a 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -9,6 +9,7 @@ import (
 
 	"git.arvados.org/arvados.git/lib/cli"
 	"git.arvados.org/arvados.git/lib/cmd"
+	"git.arvados.org/arvados.git/lib/deduplicationreport"
 	"git.arvados.org/arvados.git/lib/mount"
 )
 
@@ -52,7 +53,8 @@ var (
 		"virtual_machine":          cli.APICall,
 		"workflow":                 cli.APICall,
 
-		"mount": mount.Command,
+		"mount":                mount.Command,
+		"deduplication-report": deduplicationreport.Command,
 	})
 )
 
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index fcea2223da..ff99de75c4 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -15,6 +15,7 @@ import (
 	"git.arvados.org/arvados.git/lib/crunchrun"
 	"git.arvados.org/arvados.git/lib/dispatchcloud"
 	"git.arvados.org/arvados.git/lib/install"
+	"git.arvados.org/arvados.git/lib/recovercollection"
 	"git.arvados.org/arvados.git/services/ws"
 )
 
@@ -24,16 +25,17 @@ var (
 		"-version":  cmd.Version,
 		"--version": cmd.Version,
 
-		"boot":            boot.Command,
-		"cloudtest":       cloudtest.Command,
-		"config-check":    config.CheckCommand,
-		"config-defaults": config.DumpDefaultsCommand,
-		"config-dump":     config.DumpCommand,
-		"controller":      controller.Command,
-		"crunch-run":      crunchrun.Command,
-		"dispatch-cloud":  dispatchcloud.Command,
-		"install":         install.Command,
-		"ws":              ws.Command,
+		"boot":               boot.Command,
+		"cloudtest":          cloudtest.Command,
+		"config-check":       config.CheckCommand,
+		"config-defaults":    config.DumpDefaultsCommand,
+		"config-dump":        config.DumpCommand,
+		"controller":         controller.Command,
+		"crunch-run":         crunchrun.Command,
+		"dispatch-cloud":     dispatchcloud.Command,
+		"install":            install.Command,
+		"recover-collection": recovercollection.Command,
+		"ws":                 ws.Command,
 	})
 )
 
diff --git a/doc/_config.yml b/doc/_config.yml
index a8394300ea..be52a204c0 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -153,8 +153,9 @@ navbar:
       - admin/index.html.textile.liquid
     - Users and Groups:
       - admin/user-management.html.textile.liquid
-      - admin/reassign-ownership.html.textile.liquid
       - admin/user-management-cli.html.textile.liquid
+      - admin/reassign-ownership.html.textile.liquid
+      - admin/link-accounts.html.textile.liquid
       - admin/group-management.html.textile.liquid
       - admin/federation.html.textile.liquid
       - admin/merge-remote-account.html.textile.liquid
@@ -174,6 +175,7 @@ navbar:
       - admin/logs-table-management.html.textile.liquid
       - admin/workbench2-vocabulary.html.textile.liquid
       - admin/storage-classes.html.textile.liquid
+      - admin/recovering-deleted-collections.html.textile.liquid
     - Cloud:
       - admin/spot-instances.html.textile.liquid
       - admin/cloudtest.html.textile.liquid
@@ -187,6 +189,8 @@ navbar:
       - install/arvbox.html.textile.liquid
     - Arvados on Kubernetes:
       - install/arvados-on-kubernetes.html.textile.liquid
+      - install/arvados-on-kubernetes-minikube.html.textile.liquid
+      - install/arvados-on-kubernetes-GKE.html.textile.liquid
     - Manual installation:
       - install/install-manual-prerequisites.html.textile.liquid
       - install/packages.html.textile.liquid
@@ -207,7 +211,6 @@ navbar:
       - install/install-keep-balance.html.textile.liquid
     - User interface:
       - install/setup-login.html.textile.liquid
-      - install/install-sso.html.textile.liquid
       - install/install-workbench-app.html.textile.liquid
       - install/install-workbench2-app.html.textile.liquid
       - install/install-composer.html.textile.liquid
@@ -215,6 +218,7 @@ navbar:
       - install/install-ws.html.textile.liquid
       - install/install-arv-git-httpd.html.textile.liquid
       - install/install-shell-server.html.textile.liquid
+      - install/install-webshell.html.textile.liquid
     - Containers API:
       - install/crunch2-slurm/install-compute-node.html.textile.liquid
       - install/install-jobs-image.html.textile.liquid
@@ -225,5 +229,4 @@ navbar:
       - install/install-postgresql.html.textile.liquid
       - install/ruby.html.textile.liquid
       - install/nginx.html.textile.liquid
-      - install/google-auth.html.textile.liquid
       - install/install-docker.html.textile.liquid
diff --git a/doc/admin/federation.html.textile.liquid b/doc/admin/federation.html.textile.liquid
index b1f1506e4c..3726a6d96c 100644
--- a/doc/admin/federation.html.textile.liquid
+++ b/doc/admin/federation.html.textile.liquid
@@ -42,7 +42,7 @@ h2(#LoginCluster). Federation user management
 
 A federation of clusters can be configured to use a separate user database per cluster, or delegate a central cluster to manage the database.
 
-If clusters belong to separate organizations, each cluster will have its own user database for the members of that organization.  Through federation, a user from one organization can be granted access to the cluster of another organization.  The admin of the seond cluster controls access on a individual basis by choosing to activate or deactivate accounts from other organizations (with the default policy the value of  @ActivateUsers@).
+If clusters belong to separate organizations, each cluster will have its own user database for the members of that organization.  Through federation, a user from one organization can be granted access to the cluster of another organization.  The admin of the second cluster controls access on a individual basis by choosing to activate or deactivate accounts from other organizations (with the default policy the value of  @ActivateUsers@).
 
 On the other hand, if all clusters belong to the same organization, and users in that organization should have access to all the clusters, user management can be simplified by setting the @LoginCluster@ which manages the user database used by all other clusters in the federation.  To do this, choose one cluster in the federation which will be the 'login cluster'.  Set the the @Login.LoginCluster@ configuration value on all clusters in the federation to the cluster id of the login cluster.  After setting @LoginCluster@, restart arvados-api-server and arvados-controller.
 
diff --git a/doc/admin/link-accounts.html.textile.liquid b/doc/admin/link-accounts.html.textile.liquid
new file mode 100644
index 0000000000..d0ac6a036a
--- /dev/null
+++ b/doc/admin/link-accounts.html.textile.liquid
@@ -0,0 +1,48 @@
+---
+layout: default
+navsection: admin
+title: "Link user accounts"
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+If a user needs to log in to Arvados with a upstream account or provider, they may end up with two Arvados user accounts.  If the user still has the ability to log in with the old account, they can use the "self-serve account linking":{{site.baseurl}}/user/topics/link-accounts.html feature of workbench.  However, if the user does not have the ability to log in with both upstream accounts, the admin can also link the accounts using the command line.
+
+h3. Step 1: Determine user uuids
+
+User uuids can be determined by browsing workbench or using @arv user list@ at the command line.
+
+Account linking works by recording in the database that a log in to the "old" account should redirected and treated as a login to the "new" account.
+
+The "old" account is the Arvados account that will be redirected.
+
+The "new" account is the user that the "old" account is redirected to.  As part of account linking any Arvados records owned by the "old" account is also transferred to the "new" account.
+
+Counter-intuitively, if you do not want the account uuid of the user to change, the "new" account should be the pre-existing account, and the "old" account should be the redundant second account that was more recently created.  This means "old" and "new" are opposite from their expected chronological meaning.  In this case, the use of "old" and "new" reflect the direction of transfer of ownership -- the login was associated with the "old" user account, but will be associated with the "new" user account.
+
+In the example below, @zzzzz-tpzed-3kz0nwtjehhl0u4@ is the "old" account (the pre-existing account we want to keep) and @zzzzz-tpzed-fr97h9t4m5jffxs@ is the "new" account (the redundant account we want to merge into the existing account).
+
+h3. Step 2: Create a project
+
+Create a project owned by the "new" account that will hold any data owned by the "old" account.
+
+<pre>
+$ arv --format=uuid group create --group '{"group_class": "project", "name": "Data from old user", "owner_uuid": "zzzzz-tpzed-fr97h9t4m5jffxs"}'
+zzzzz-j7d0g-mczqiguhil13083
+</pre>
+
+h3. Step 3: Merge "old" user to "new" user
+
+The @user merge@ method redirects login and reassigns data from the "old" account to the "new" account.
+
+<pre>
+$ arv user merge  --redirect-to-new-user \
+  --old-user-uuid=zzzzz-tpzed-3kz0nwtjehhl0u4 \
+  --new-user-uuid=zzzzz-tpzed-fr97h9t4m5jffxs \
+  --new-owner-uuid=zzzzz-j7d0g-mczqiguhil13083 \
+</pre>
+
+Note that authorization credentials (API tokens, ssh keys) are also transferred to the "new" account, so credentials used to access the "old" account work with the "new" account.
diff --git a/doc/admin/metrics.html.textile.liquid b/doc/admin/metrics.html.textile.liquid
index a6a0862c4f..1d6b87da62 100644
--- a/doc/admin/metrics.html.textile.liquid
+++ b/doc/admin/metrics.html.textile.liquid
@@ -42,7 +42,6 @@ table(table table-bordered table-condensed table-hover).
 |keepstore|✓|
 |keep-balance|✓|
 |keep-web|✓|
-|sso-provider||
 |workbench1||
 |workbench2||
 
diff --git a/doc/admin/migrating-providers.html.textile.liquid b/doc/admin/migrating-providers.html.textile.liquid
index 6dd0d866e7..b684111f95 100644
--- a/doc/admin/migrating-providers.html.textile.liquid
+++ b/doc/admin/migrating-providers.html.textile.liquid
@@ -9,37 +9,8 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This page describes how to enable users to use more than one upstream identity provider to log into the same Arvados account.  This can be used to migrate account providers, for example, from LDAP to Google.  In order to do this, users must be able to log into both the "old" and "new" providers.
+When a user logs in to Arvados, their email address (as returned by the authentication provider) is used as the primary key for their Arvados account.
 
-h2. Configure multiple or alternate provider in SSO
+If you reconfigure Arvados to use a different authentication provider after some users have created accounts, you should either ensure the new provider returns the same email addresses as the old one, or update your Arvados users' @email@ attributes to match the email addresses returned by the new provider.
 
-In @application.yml@ for the SSO server, you can enable both @google_oauth2@ and @ldap@ providers:
-
-<pre>
-production:
-  google_oauth2_client_id: abcd
-  google_oauth2_client_secret: abcd
-
-  use_ldap:
-    title: Example LDAP
-    host: ldap.example.com
-    port: 636
-    method: ssl
-    base: "ou=Users, dc=example, dc=com"
-    uid: uid
-    username: uid
-</pre>
-
-Restart the SSO server after changing the configuration.
-
-h2. Matching on email address
-
-If the new account provider supplies an email address (primary or alternate) that matches an existing user account, the user will be logged into that account.  No further migration is necessary, and the old provider can be removed from the SSO configuration.
-
-h2. Link accounts
-
-If the new provider cannot provide matching email addresses, users will have to migrate manually by "linking accounts":{{site.baseurl}}/user/topics/link-accounts.html
-
-After linking accounts, users can use the new provider to access their existing Arvados account.
-
-Once all users have migrated, the old account provider can be removed from the SSO configuration.
+Otherwise, next time users log in, they will be given new accounts instead of logging in to their existing accounts.
diff --git a/doc/admin/recovering-deleted-collections.html.textile.liquid b/doc/admin/recovering-deleted-collections.html.textile.liquid
new file mode 100644
index 0000000000..59c576ce03
--- /dev/null
+++ b/doc/admin/recovering-deleted-collections.html.textile.liquid
@@ -0,0 +1,37 @@
+---
+layout: default
+navsection: admin
+title: Recovering deleted collections
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+In some cases, it is possible to recover files that have been lost by modifying or deleting a collection.
+
+Possibility of recovery depends on many factors, including:
+* Whether the collection manifest is still available, e.g., in an audit log entry
+* Whether the data blocks are also referenced by other collections
+* Whether the data blocks have been unreferenced long enough to be marked for deletion/trash by keep-balance
+* Blob signature TTL, trash lifetime, trash check interval, and other config settings
+
+To attempt recovery of a previous version of a deleted/modified collection, use the @arvados-server recover-collection@ command. It should be run on one of your server nodes where the @arvados-server@ package is installed and the @/etc/arvados/config.yml@ file is up to date.
+
+Specify the collection you want to recover by passing either the UUID of an audit log entry, or a file containing the manifest.
+
+If recovery is successful, the @recover-collection@ program saves the recovered data a new collection belonging to the system user, and prints the new collection's UUID on stdout.
+
+<pre>
+# arvados-server recover-collection 9tee4-57u5n-nb5awmk1pahac2t
+INFO[2020-06-05T19:52:29.557761245Z] loaded log entry                              logged_event_time="2020-06-05 16:48:01.438791 +0000 UTC" logged_event_type=update old_collection_uuid=9tee4-4zz18-1ex26g95epmgw5w src=9tee4-57u5n-nb5awmk1pahac2t
+INFO[2020-06-05T19:52:29.642145127Z] recovery succeeded                            UUID=9tee4-4zz18-5trfp4k4xxg97f1 src=9tee4-57u5n-nb5awmk1pahac2t
+9tee4-4zz18-5trfp4k4xxg97f1
+INFO[2020-06-05T19:52:29.644699436Z] exiting
+</pre>
+
+In this example, the original data has been restored and saved in a new collection with UUID @9tee4-4zz18-5trfp4k4xxg97f1@.
+
+For more options, run @arvados-server recover-collection -help@.
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 23d7120438..9cddce5fe6 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -34,11 +34,54 @@ TODO: extract this information based on git commit messages and generate changel
 <div class="releasenotes">
 </notextile>
 
-h2(#master). development master (as of 2020-02-07)
+h2(#master). development master (as of 2020-06-17)
 
 "Upgrading from 2.0.0":#v2_0_0
 
-None in current development master.
+h3. Removing sso-provider
+
+The SSO (single sign-on) component is deprecated and will not be supported in future releases. Existing configurations will continue to work in this release, but you should switch to one of the built-in authentication mechanisms as soon as possible. See "setting up web based login":{{site.baseurl}}/install/setup-login.html for details.
+
+After migrating your configuration, uninstall the @arvados-sso-provider@ package.
+
+h3. S3 signatures
+
+Keepstore now uses "V4 signatures":https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html by default for S3 requests. If you are using Amazon S3, no action is needed; all regions support V4 signatures. If you are using a different S3-compatible service that does not support V4 signatures, add @V2Signature: true@ to your volume driver parameters to preserve the old behavior. See "configuring S3 object storage":{{site.baseurl}}/install/configure-s3-object-storage.html.
+
+h3. New permission system constraints
+
+Some constraints on the permission system have been added, in particular @role@ and @project@ group types now have distinct behavior. These constraints were already de-facto imposed by the Workbench UI, so on most installations the only effect of this migration will be to reassign @role@ groups to the system user and create a @can_manage@ permission link for the previous owner.
+
+# The @group_class@ field must be either @role@ or @project@. Invalid group_class are migrated to @role@.
+# A @role@ cannot own things. Anything owned by a role is migrated to a @can_manage@ link and reassigned to the system user.
+# Only @role@ and @user@ can have outgoing permission links. Permission links originating from projects are deleted by the migration.
+# A @role@ is always owned by the system_user. When a group is created, it creates a @can_manage@ link for the object that would have been assigned to @owner_uuid@.  Migration adds @can_manage@ links and reassigns roles to the system user.  This also has the effect of requiring that all @role@ groups have unique names on the system.  If there is a name collision during migration, roles will be renamed to ensure they are unique.
+# A permission link can have the permission level (@name@) updated but not @head_uuid@, @tail_uuid@ or @link_class@.
+
+The @arvados-sync-groups@ tool has been updated to reflect these constraints, so it is important to use the version of @arvados-sync-groups@ that matches the API server version.
+
+Before upgrading, use the following commands to find out which groups and permissions in your database will be automatically modified or deleted during the upgrade.
+
+To determine which groups have invalid @group_class@ (these will be migrated to @role@ groups):
+
+<pre>
+arv group list --filters '[["group_class", "not in", ["project", "role"]]]'
+</pre>
+
+To list all @role@ groups, which will be reassigned to the system user (unless @owner_uuid@ is already the system user):
+
+<pre>
+arv group list --filters '[["group_class", "=", "role"]]'
+</pre>
+
+To list which @project@ groups have outgoing permission links (such links are now invalid and will be deleted by the migration):
+
+<pre>
+for uuid in $(arv link list --filters '[["link_class", "=", "permission"], ["tail_uuid", "like", "%-j7d0g-%"]]' |
+              jq -r .items[].tail_uuid | sort | uniq) ; do
+   arv group list --filters '[["group_class", "=", "project"], ["uuid", "=", "'$uuid'"]]' | jq .items
+done
+</pre>
 
 h2(#v2_0_0). v2.0.0 (2020-02-07)
 
diff --git a/doc/admin/user-management.html.textile.liquid b/doc/admin/user-management.html.textile.liquid
index 177abd8dbe..9e53775ed4 100644
--- a/doc/admin/user-management.html.textile.liquid
+++ b/doc/admin/user-management.html.textile.liquid
@@ -24,7 +24,7 @@ After completing the log in and authentication process, the API server receives
 
 If a provider identifier is given, the API server searches for a matching user record.
 
-If a provider identifier is not given, no match is found, it next searches by primary email and then alternate email address.  This enables "provider migration":migrating-providers.html and a "pre-activated accounts.":#pre-activated
+If a provider identifier is not given, no match is found, it next searches by primary email and then alternate email address.  This enables "provider migration":migrating-providers.html and "pre-activated accounts.":#pre-activated
 
 If no user account is found, a new user account is created with the information from the identity provider.
 
diff --git a/doc/api/methods/users.html.textile.liquid b/doc/api/methods/users.html.textile.liquid
index 4c33f2afe8..cde189d6ff 100644
--- a/doc/api/methods/users.html.textile.liquid
+++ b/doc/api/methods/users.html.textile.liquid
@@ -154,3 +154,21 @@ Arguments:
 table(table table-bordered table-condensed).
 |_. Argument |_. Type |_. Description |_. Location |_. Example |
 {background:#ccffcc}.|uuid|string|The UUID of the User in question.|path||
+
+h3. merge
+
+Transfer ownership of data from the "old" user account to the "new" user account.  When @redirect_to_new_user@ is @true@ this also causes logins to the "old" account to be redirected to the "new" account.  The "old" user account that was redirected becomes invisible in user listings.
+
+See "Merge user accounts":{{site.baseurl}}/admin/link-accounts.html , "Reassign user data ownership":{{site.baseurl}}/admin/reassign-ownership.html and "Linking alternate login accounts":{{site.baseurl}}/user/topics/link-accounts.html for examples of how this method is used.
+
+Must supply either @new_user_token@ (the currently authorized user will be the "old" user), or both @new_user_uuid@ and @old_user_uuid@ (the currently authorized user must be an admin).
+
+Arguments:
+
+table(table table-bordered table-condensed).
+|_. Argument |_. Type |_. Description |_. Location |_. Example |
+|new_user_token|string|A valid token for the "new" user|query||
+|new_user_uuid|uuid|The uuid of the "new" account|query||
+|old_user_uuid|uuid|The uuid of the "old" account|query||
+|new_owner_uuid|uuid|The uuid of a project to which objects owned by the "old" user will be reassigned.|query||
+|redirect_to_new_user|boolean|If true, also redirect login and reassign authorization credentials from "old" user to the "new" user|query||
diff --git a/doc/api/permission-model.html.textile.liquid b/doc/api/permission-model.html.textile.liquid
index 1f08ea4195..7f10521299 100644
--- a/doc/api/permission-model.html.textile.liquid
+++ b/doc/api/permission-model.html.textile.liquid
@@ -10,59 +10,89 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-* There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*.
-** *none* is the default state when there are no other permission grants.
-*** the object is not included in any list query response.
-*** direct queries of the object by uuid return 404 Not Found.
-*** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found.
-** *can_read* grants read-only access to the record.  Attempting to update or delete the record returns an error.  *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions.
-** *can_write* permits changes to the record (but not permission links).  *can_write* permits the user to delete the object.  *can_write* also implies *can_read*.
-** *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@.  *can_manage* also implies *can_write* and *can_read*.
+There are four levels of permission: *none*, *can_read*, *can_write*, and *can_manage*.
+
+* *none* is the default state when there are no other permission grants.
+** the object is not included in any list query response.
+** direct queries of the object by uuid return 404 Not Found.
+** Link objects require valid identifiers in @head_uuid@ and @tail_uuid@, so an attempt to create a Link that references an unreadable object will return an error indicating the object is not found.
+* *can_read* grants read-only access to the record.  Attempting to update or delete the record returns an error.
+** *can_read* does not allow a reader to see any permission grants on the object except the object's owner_uuid and the reader's own permissions.
+* *can_write* permits changes to the record, including changing ownership and deleting the object.
+** *can_write* cannot read, create, update or delete permission links associated with the object.
+** *can_write* also implies *can_read*.
+* *can_manage* permits the user to read, create, update and delete permission links whose @head_uuid@ is this object's @uuid@.
+** *can_manage* also implies *can_write* and *can_read*.
 
 h2. Ownership
 
-* All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group".
-* The User or Group specified by @owner_uuid@ has *can_manage* permission on the object.
-** This permission is one way: A User or Group's @owner_uuid@ being equal to @X@ does not imply any permission for that User/Group to read, write, or manage an object whose @uuid@ is equal to @X@.
-* Applications should represent each object as belonging to, or being "inside", the Group/User referenced by its @owner_uuid@.
-** A "project" is a subtype of Group that is treated as a "Project" in Workbench, and as a directory by @arv-mount@.
-** A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
-* To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner.
+All Arvados objects have an @owner_uuid@ field. Valid uuid types for @owner_uuid@ are "User" and "Group".  For Group, the @group_class@ must be a "project".
+
+The User or Group specified by @owner_uuid@ has *can_manage* permission on the object.  This permission is one way: an object that is owned does not get any special permissions on the User or Group that owns it.
+
+To change the @owner_uuid@ field, it is necessary to have @can_write@ permission on both the current owner and the new owner.
 
 h2(#links). Permission links
 
-A link object with
+A permission link is a link object with:
 
 * @owner_uuid@ of the system user.
 * @link_class@ "permission"
 * @name@ one of *can_read*, *can_write* or *can_manage*
 * @head_uuid@ of some Arvados object
-* @tail_uuid@ of a User or Group
+* @tail_uuid@ of a User or Group.  For Group, the @group_class@ must be a "role".
 
-grants the @name@ permission for @tail_uuid@ accessing @head_uuid@
+This grants the permission in @name@ for @tail_uuid@ accessing @head_uuid@.
 
-* If a User has *can_manage* permission on some object, this grants permission to read, create, update and delete permission links where the @head_uuid@ is the object under management.
+If a User has *can_manage* permission on some object, the user has the ability to read, create, update and delete permission links with @head_uuid@ of the managed object.  In other words, the user has the ability to modify the permission grants on the object.
 
 h3. Transitive permissions
 
-Permissions can be obtained indirectly through Groups.
-* If a User X *can_read* Group A, and Group A *can_read* Object B, then User X *can_read* Object B.
+Permissions can be obtained indirectly through nested ownership (*can_manage*) or by following multiple permission links.
+
+* If a User X owns project A, and project A owns project B, then User X *can_manage* project B.
+* If a User X *can_read* role A, and role A *can_read* Object B, then User X *can_read* Object B.
 * Permissions are narrowed to the least powerful permission on the path.
-** If User X *can_write* Group A, and Group A *can_read* Object B, then User X *can_read* Object B.
-** If User X *can_read* Group A, and Group A *can_write* Object B, then User X *can_read* Object B.
+** If User X *can_write* role A, and role A *can_read* Object B, then User X *can_read* Object B.
+** If User X *can_read* role A, and role A *can_write* Object B, then User X *can_read* Object B.
+
+h2. Projects and Roles
+
+A "project" is a subtype of Group that is displayed as a "Project" in Workbench, and as a directory by @arv-mount@.
+* A project can own things (appear in @owner_uuid@)
+* A project can be owned by a user or another project.
+* The name of a project is unique only among projects with the same owner_uuid.
+* Projects can be targets (@head_uuid@) of permission links, but not origins (@tail_uuid@).  Putting a project in a @tail_uuid@ field is an error.
+
+A "role" is a subtype of Group that is treated in Workbench as a group of users who have permissions in common (typically an organizational group).
+* A role cannot own things (cannot appear in @owner_uuid@).  Putting a role in an @owner_uuid@ field is an error.
+* All roles are owned by the system user.
+* The name of a role is unique across a single Arvados cluster.
+* Roles can be both targets (@head_uuid@) and origins (@tail_uuid@) of permission links.
+
+h3. Access through Roles
 
-h2. Group Membership
+A "role" consists of a set of users or other roles that have that role, and a set of permissions (primarily read/write/manage access to projects) the role grants.
 
-Group membership is determined by whether the group has *can_read* permission on an object.  If a group G *can_read* an object A, then we say A is a member of G.
+If there is a permission link stating that user A *can_write* role R, then we say A has role R.  This means user A has up to *can_write* access to everything the role has access to.
 
-For some kinds of groups, like roles, it is natural for users who are members of a group to also have *can_manage* permission on the group, i.e., G *can_read* A  and A *can_manage* G ("A can do anything G can do"). However, this is not necessary: A can be a member of a group while being unable to even read it.
+Because permissions are one-way, the links A *can_write* R and B *can_write* R does not imply that user A and B will be able to see each other.  For users in a role to see each other, read permission should be added going in the opposite direction: R *can_read* A and R *can_read* B.
+
+If a user needs to be able to manipulate permissions of objects that are accessed through the role (for example, to share project P with a user outside the role), then role R must have *can_manage* permission on project P (R *can_manage* P) and the user must be granted *can_manage* permission on R (A *can_manage* R).
 
 h2. Special cases
 
-* Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read).  To retain the integrity of the log, the log table should deny all update or delete operations.
-* Permission links where @tail_uuid@ is a User permit @can_read@ on the link by that user.  (User can discover her own permission grants.)
-* *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks)
-* If User or Group X *can_FOO* Group A, and Group A *can_manage* User B, then X *can_FOO* _everything that User B can_FOO_.
+Log table objects are additionally readable based on whether the User has *can_read* permission on @object_uuid@ (User can access log history about objects it can read).  To retain the integrity of the log, the log table denies all update or delete operations.
+
+Permission links where @tail_uuid@ is a User allow *can_read* on the link record by that user (User can discover her own permission grants.)
+
+At least *can_read* on a Collection grants permission to read the blocks that make up the collection (API server returns signed blocks).
+
+A user can only read a container record if the user has read permission to a container_request with that container_uuid.
+
+*can_read* and *can_write* access on a user grants access to the user record, but not anything owned by the user.
+*can_manage* access to a user grants can_manage access to the user, _and everything owned by that user_ .
+If a user A *can_read* role R, and role R *can_manage* user B, then user A *can_read* user B _and everything owned by that user_ .
 
 h2(#system). System user and group
 
@@ -70,7 +100,9 @@ A privileged user account exists for the use by internal Arvados components.  Th
 
 h2. Anoymous user and group
 
-An Arvados site may be configured to allow users to browse resources without requiring a login.  In this case, permissions for non-logged-in users are associated with the "anonymous" user.  To make objects visible to the public, they can be shared with the "anonymous" group.  The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@.  The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@.
+An Arvados site may be configured to allow users to browse resources without requiring a login.  In this case, permissions for non-logged-in users are associated with the "anonymous" user.  To make objects visible to anyone (both logged-in and non-logged-in users), they can be shared with the "anonymous" role.  Note that objects shared with the "anonymous" user will only be visible to non-logged-in users!
+
+The anonymous user uuid is @{siteprefix}-tpzed-anonymouspublic@.  The anonymous group uuid is @{siteprefix}-j7d0g-anonymouspublic@.
 
 h2. Example
 
diff --git a/doc/architecture/index.html.textile.liquid b/doc/architecture/index.html.textile.liquid
index c7ea3268e1..705048cd62 100644
--- a/doc/architecture/index.html.textile.liquid
+++ b/doc/architecture/index.html.textile.liquid
@@ -56,4 +56,4 @@ table(table table-bordered table-condensed).
 |keep-block-check|Given a list of keep block locators, check that each block exists on one of the configured keepstore servers and verify the block hash.|
 |keep-exercise|Benchmarking tool to test throughput and reliability of keepstores under various usage patterns.|
 |keep-rsync|Get lists of blocks from two clusters, copy blocks which exist on source cluster but are missing from destination cluster.|
-|sync-groups|Take a CSV file listing (group, username) pairs and synchronize membership in Arvados groups.|
+|sync-groups|Take a CSV file listing with (group, user, permission) records and synchronize membership in Arvados groups.|
diff --git a/doc/examples/config/zzzzz.yml b/doc/examples/config/zzzzz.yml
index c63550edf7..d1e1336d54 100644
--- a/doc/examples/config/zzzzz.yml
+++ b/doc/examples/config/zzzzz.yml
@@ -1,3 +1,4 @@
+AutoReloadConfig: true
 Clusters:
   zzzzz:
     ManagementToken: e687950a23c3a9bceec28c6223a06c79
diff --git a/doc/index.html.liquid b/doc/index.html.liquid
index f81cd59017..c7fd46eb22 100644
--- a/doc/index.html.liquid
+++ b/doc/index.html.liquid
@@ -34,7 +34,8 @@ SPDX-License-Identifier: CC-BY-SA-3.0
       <a name="Support"></a>
       <p><strong>Support and Community</strong></p>
 
-      <p>The <a href="https://gitter.im/arvados/community">arvados community channel</a> at gitter.im is available for live discussion and community support.  There is also a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>. 
+      <p>Interact with the Arvados community on the <a href="https://forum.arvados.org">Arvados Forum</a>
+	and the <a href="https://gitter.im/arvados/community">arvados/community</a> channel at gitter.im.
       </p>
 
       <p>Curii Corporation provides managed Arvados installations as well as commercial support for Arvados. Please contact <a href="mailto:info@curii.com">info@curii.com</a> for more information.</p>
diff --git a/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid b/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
index 88b2d5730d..0801b7d4e3 100644
--- a/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes-GKE.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: installguide
-title: Arvados on Kubernetes - Google Kubernetes Engine
+title: Arvados on GKE
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,7 +9,13 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This page documents the setup of the prerequisites to run the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
+This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Google Kubernetes Engine@ (GKE).
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+h2. Prerequisites
 
 h3. Install tooling
 
@@ -27,12 +33,12 @@ Install @helm@:
 
 * Follow the instructions at "https://docs.helm.sh/using_helm/#installing-helm":https://docs.helm.sh/using_helm/#installing-helm
 
-h3. Boot the GKE cluster
+h3. Create the GKE cluster
 
 This can be done via the "cloud console":https://console.cloud.google.com/kubernetes/ or via the command line:
 
 <pre>
-$ gcloud container clusters create <CLUSTERNAME> --zone us-central1-a --machine-type n1-standard-2 --cluster-version 1.10
+$ gcloud container clusters create <CLUSTERNAME> --zone us-central1-a --machine-type n1-standard-2 --cluster-version 1.15
 </pre>
 
 It takes a few minutes for the cluster to be initialized.
@@ -59,4 +65,91 @@ Test the connection:
 $ kubectl get nodes
 </pre>
 
-Now proceed to the "Initialize helm on the Kubernetes cluster":/install/arvados-on-kubernetes.html#helm section.
+Test @helm@ by running
+
+<pre>
+$ helm ls
+</pre>
+
+There should be no errors. The command will return nothing.
+
+h2(#git). Clone the repository
+
+Clone the repository and nagivate to the @arvados-k8s/charts/arvados@ directory:
+
+<pre>
+$ git clone https://github.com/arvados/arvados-k8s.git
+$ cd arvados-k8s/charts/arvados
+</pre>
+
+h2(#Start). Start the Arvados cluster
+
+Next, determine the IP address that the Arvados cluster will use to expose its API, Workbench, etc. If you want this Arvados cluster to be reachable from places other than the local machine, the IP address will need to be routable as appropriate.
+
+<pre>
+$ ./cert-gen.sh <IP ADDRESS>
+</pre>
+
+The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
+
+<pre>
+  adminUserEmail
+  adminUserPassword
+  superUserSecret
+  anonymousUserSecret
+</pre>
+
+Now start the Arvados cluster:
+
+<pre>
+$ helm install arvados . --set externalIP=<IP ADDRESS>
+</pre>
+
+At this point, you can use kubectl to see the Arvados cluster boot:
+
+<pre>
+$ kubectl get pods
+$ kubectl get svc
+</pre>
+
+After a few minutes, there shouldn't be any services listed with a 'Pending' external IP address. At that point you can access Arvados Workbench at the IP address specified
+
+* https://&lt;IP ADDRESS&gt;
+
+with the username and password specified in the @values.yaml@ file.
+
+Alternatively, use the Arvados cli tools or SDKs. First set the environment variables:
+
+<pre>
+$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
+$ export ARVADOS_API_HOST=<STATIC IP>:444
+$ export ARVADOS_API_HOST_INSECURE=true
+</pre>
+
+Test access with:
+
+<pre>
+$ arv user current
+</pre>
+
+h2(#reload). Reload
+
+If you make changes to the Helm chart (e.g. to @values.yaml@), you can reload Arvados with
+
+<pre>
+$ helm upgrade arvados .
+</pre>
+
+h2. Shut down
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+<pre>
+$ helm del arvados
+</pre>
+
+<pre>
+$ gcloud container clusters delete <CLUSTERNAME> --zone us-central1-a
+</pre>
diff --git a/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid b/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
index 132b443dff..86aaf08f96 100644
--- a/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes-minikube.html.textile.liquid
@@ -1,7 +1,7 @@
 ---
 layout: default
 navsection: installguide
-title: Arvados on Kubernetes - Minikube
+title: Arvados on Minikube
 ...
 {% comment %}
 Copyright (C) The Arvados Authors. All rights reserved.
@@ -9,7 +9,13 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-This page documents the setup of the prerequisites to run the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
+This page documents setting up and running the "Arvados on Kubernetes":/install/arvados-on-kubernetes.html @Helm@ chart on @Minikube@.
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+h2. Prerequisites
 
 h3. Install tooling
 
@@ -31,4 +37,100 @@ Test the connection:
 $ kubectl get nodes
 </pre>
 
-Now proceed to the "Initialize helm on the Kubernetes cluster":/install/arvados-on-kubernetes.html#helm section.
+Test @helm@ by running
+
+<pre>
+$ helm ls
+</pre>
+
+There should be no errors. The command will return nothing.
+
+h2(#git). Clone the repository
+
+Clone the repository and nagivate to the @arvados-k8s/charts/arvados@ directory:
+
+<pre>
+$ git clone https://github.com/arvados/arvados-k8s.git
+$ cd arvados-k8s/charts/arvados
+</pre>
+
+h2(#Start). Start the Arvados cluster
+
+All Arvados services will be accessible on Minikube's IP address. This will be a local IP address, you can see what it is by running
+
+<pre>
+$ minikube ip
+192.168.39.15
+</pre>
+
+Generate self-signed SSL certificates for the Arvados services:
+
+<pre>
+$ ./cert-gen.sh `minikube ip`
+</pre>
+
+The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
+
+<pre>
+  adminUserEmail
+  adminUserPassword
+  superUserSecret
+  anonymousUserSecret
+</pre>
+
+Now start the Arvados cluster:
+
+<pre>
+$ helm install arvados . --set externalIP=`minikube ip`
+</pre>
+
+And update the Kubernetes services to have the Minikube IP as their 'external' IP:
+
+<pre>
+$ ./minikube-external-ip.sh
+</pre>
+
+At this point, you can use kubectl to see the Arvados cluster boot:
+
+<pre>
+$ kubectl get pods
+$ kubectl get svc
+</pre>
+
+After a few minutes, you can access Arvados Workbench at the Minikube IP address:
+
+* https://&lt;MINIKUBE IP&gt;
+
+with the username and password specified in the @values.yaml@ file.
+
+Alternatively, use the Arvados cli tools or SDKs. First set the environment variables:
+
+<pre>
+$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
+$ export ARVADOS_API_HOST=<MINIKUBE IP>:444
+$ export ARVADOS_API_HOST_INSECURE=true
+</pre>
+
+Test access with:
+
+<pre>
+$ arv user current
+</pre>
+
+h2(#reload). Reload
+
+If you make changes to the Helm chart (e.g. to @values.yaml@), you can reload Arvados with
+
+<pre>
+$ helm upgrade arvados .
+</pre>
+
+h2. Shut down
+
+{% include 'notebox_begin_warning' %}
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
+{% include 'notebox_end' %}
+
+<pre>
+$ helm del arvados
+</pre>
diff --git a/doc/install/arvados-on-kubernetes.html.textile.liquid b/doc/install/arvados-on-kubernetes.html.textile.liquid
index 08b2c7329e..ff52aa171f 100644
--- a/doc/install/arvados-on-kubernetes.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes.html.textile.liquid
@@ -9,125 +9,28 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-Arvados on Kubernetes is implemented as a Helm Chart.
+Arvados on Kubernetes is implemented as a @Helm 3@ chart.
 
 {% include 'notebox_begin_warning' %}
-This Helm Chart does not retain any state after it is deleted. An Arvados cluster created with this Helm Chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
+This Helm chart does not retain any state after it is deleted. An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down. This will be fixed in a future version.
 {% include 'notebox_end' %}
 
 h2(#overview). Overview
 
-This Helm Chart provides a basic, small Arvados cluster.
+This Helm chart provides a basic, small Arvados cluster.
 
 Current limitations, to be addressed in the future:
 
-* An Arvados cluster created with this Helm Chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down.
-* No dynamic scaling of compute nodes (but you can adjust @values.yaml@ and "reload the Helm Chart":#reload
+* An Arvados cluster created with this Helm chart is entirely ephemeral, and all data stored on the cluster will be deleted when it is shut down.
+* No dynamic scaling of compute nodes (but you can adjust @values.yaml@ and reload the Helm chart)
 * All compute nodes are the same size
 * Compute nodes have no cpu/memory/disk constraints yet
 * No git server
 
 h2. Requirements
 
-* Kubernetes 1.10+ cluster with at least 3 nodes, 2 or more cores per node
-* @kubectl@ and @helm@ installed locally, and able to connect to your Kubernetes cluster
+* Minikube or Google Kubernetes Engine (Kubernetes 1.10+ with at least 3 nodes, 2+ cores per node)
+* @kubectl@ and @Helm 3@ installed locally, and able to connect to your Kubernetes cluster
 
-If you do not have a Kubernetes cluster already set up, you can use "Google Kubernetes Engine":/install/arvados-on-kubernetes-GKE.html for multi-node development and testing or "another Kubernetes solution":https://kubernetes.io/docs/setup/pick-right-solution/. Minikube is not supported yet.
+Please refer to "Arvados on Minikube":/install/arvados-on-kubernetes-minikube.html or "Arvados on GKE":/install/arvados-on-kubernetes-GKE.html for detailed installation instructions.
 
-h2(#helm). Initialize helm on the Kubernetes cluster
-
-If you already have helm running on the Kubernetes cluster, proceed directly to "Start the Arvados cluster":#Start below.
-
-<pre>
-$ helm init
-$ kubectl create serviceaccount --namespace kube-system tiller
-$ kubectl create clusterrolebinding tiller-cluster-rule --clusterrole=cluster-admin --serviceaccount=kube-system:tiller
-$ kubectl patch deploy --namespace kube-system tiller-deploy -p '{"spec":{"template":{"spec":{"serviceAccount":"tiller"}}}}'
-</pre>
-
-Test @helm@ by running
-
-<pre>
-$ helm ls
-</pre>
-
-There should be no errors. The command will return nothing.
-
-h2(#git). Clone the repository
-
-Clone the repository and nagivate to the @arvados-kubernetes/charts/arvados@ directory:
-
-<pre>
-$ git clone https://github.com/arvados/arvados-kubernetes.git
-$ cd arvados-kubernetes/charts/arvados
-</pre>
-
-h2(#Start). Start the Arvados cluster
-
-Next, determine the IP address that the Arvados cluster will use to expose its API, Workbench, etc. If you want this Arvados cluster to be reachable from places other than the local machine, the IP address will need to be routable as appropriate.
-
-<pre>
-$ ./cert-gen.sh <IP ADDRESS>
-</pre>
-
-The @values.yaml@ file contains a number of variables that can be modified. At a minimum, review and/or modify the values for
-
-<pre>
-  adminUserEmail
-  adminUserPassword
-  superUserSecret
-  anonymousUserSecret
-</pre>
-
-Now start the Arvados cluster:
-
-<pre>
-$ helm install --name arvados . --set externalIP=<IP ADDRESS>
-</pre>
-
-At this point, you can use kubectl to see the Arvados cluster boot:
-
-<pre>
-$ kubectl get pods
-$ kubectl get svc
-</pre>
-
-After a few minutes, you can access Arvados Workbench at the IP address specified
-
-* https://&lt;IP ADDRESS&gt;
-
-with the username and password specified in the @values.yaml@ file.
-
-Alternatively, use the Arvados cli tools or SDKs:
-
-Set the environment variables:
-
-<pre>
-$ export ARVADOS_API_TOKEN=<superUserSecret from values.yaml>
-$ export ARVADOS_API_HOST=<STATIC IP>:444
-$ export ARVADOS_API_HOST_INSECURE=true
-</pre>
-
-Test access with:
-
-<pre>
-$ arv user current
-</pre>
-
-h2(#reload). Reload
-
-If you make changes to the Helm Chart (e.g. to @values.yaml@), you can reload Arvados with
-
-<pre>
-$ helm upgrade arvados .
-</pre>
-
-h2. Shut down
-
-{% include 'notebox_begin_warning' %}
-This Helm Chart does not retain any state after it is deleted. An Arvados cluster created with this Helm Chart is entirely ephemeral, and <strong>all data stored on the Arvados cluster will be deleted</strong> when it is shut down. This will be fixed in a future version.
-{% include 'notebox_end' %}
-
-<pre>
-$ helm del arvados --purge
-</pre>
diff --git a/doc/install/configure-azure-blob-storage.html.textile.liquid b/doc/install/configure-azure-blob-storage.html.textile.liquid
index 8c5098abe3..2ccec586e4 100644
--- a/doc/install/configure-azure-blob-storage.html.textile.liquid
+++ b/doc/install/configure-azure-blob-storage.html.textile.liquid
@@ -67,8 +67,8 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
           # If the AccessViaHosts section is empty or omitted, all
           # keepstore servers will have read/write access to the
           # volume.
-          "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
-          "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
+          "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+          "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {ReadOnly: true}
 
         Driver: <span class="userinput">Azure</span>
         DriverParameters:
diff --git a/doc/install/configure-s3-object-storage.html.textile.liquid b/doc/install/configure-s3-object-storage.html.textile.liquid
index 9708ea5cd1..b960ac1fda 100644
--- a/doc/install/configure-s3-object-storage.html.textile.liquid
+++ b/doc/install/configure-s3-object-storage.html.textile.liquid
@@ -25,8 +25,8 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
           # If the AccessViaHosts section is empty or omitted, all
           # keepstore servers will have read/write access to the
           # volume.
-          "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
-          "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {ReadOnly: true}
+          "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+          "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {ReadOnly: true}
 
         Driver: <span class="userinput">S3</span>
         DriverParameters:
@@ -59,6 +59,11 @@ Volumes are configured in the @Volumes@ section of the cluster configuration fil
           # declaration.
           LocationConstraint: false
 
+          # Use V2 signatures instead of the default V4. Amazon S3
+          # supports V4 signatures in all regions, but this option
+          # might be needed for other S3-compatible services.
+          V2Signature: false
+
           # Requested page size for "list bucket contents" requests.
           IndexPageSize: 1000
 
diff --git a/doc/install/google-auth.html.textile.liquid b/doc/install/google-auth.html.textile.liquid
deleted file mode 100644
index fad10ff358..0000000000
--- a/doc/install/google-auth.html.textile.liquid
+++ /dev/null
@@ -1,27 +0,0 @@
----
-layout: default
-navsection: installguide
-title: Setting up Google auth
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-In order to use Google for authentication, you must use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
-
-# Go to the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> and select or create a project; this will take you to the project page.
-# Click on *+ Enable APIs and Services*
-## Search for *People API* and click on *Enable API*.
-# Navigate back to the main "APIs & Services" page
-# On the sidebar, click on *OAuth consent screen*
-## On consent screen settings, enter your identifying details
-## Under *Authorized domains* add @example.com@
-## Click on *Save*.
-# On the sidebar, click on *Credentials*; then click on *Create credentials*&rarr;*OAuth Client ID*
-# Under *Application type* select *Web application*.
-# You must set the authorization origins.  Edit @auth.example.com@ to the appropriate hostname that you will use to access the SSO service:
-## JavaScript origin should be @https://ClusterID.example.com/@ (using Arvados-controller based login) or @https://auth.example.com/@ (for the SSO server)
-## Redirect URI should be @https://ClusterID.example.com/login@ (using Arvados-controller based login) or @https://auth.example.com/users/auth/google_oauth2/callback@ (for the SSO server)
-# Copy the values of *Client ID* and *Client secret* from the Google Developers Console and add them to the appropriate configuration.
diff --git a/doc/install/install-api-server.html.textile.liquid b/doc/install/install-api-server.html.textile.liquid
index e64c382669..b8442eb060 100644
--- a/doc/install/install-api-server.html.textile.liquid
+++ b/doc/install/install-api-server.html.textile.liquid
@@ -142,10 +142,9 @@ server {
   # This configures the public https port that clients will actually connect to,
   # the request is reverse proxied to the upstream 'controller'
 
-  listen       *:443 ssl;
-  server_name  <span class="userinput">xxxxx.example.com</span>;
+  listen       443 ssl;
+  server_name  <span class="userinput">ClusterID.example.com</span>;
 
-  ssl on;
   ssl_certificate     <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
   ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
 
diff --git a/doc/install/install-arv-git-httpd.html.textile.liquid b/doc/install/install-arv-git-httpd.html.textile.liquid
index d501f46b7a..3d70fc4de9 100644
--- a/doc/install/install-arv-git-httpd.html.textile.liquid
+++ b/doc/install/install-arv-git-httpd.html.textile.liquid
@@ -224,12 +224,11 @@ Use a text editor to create a new file @/etc/nginx/conf.d/arvados-git.conf@ with
   server                  127.0.0.1:<span class="userinput">9001</span>;
 }
 server {
-  listen                  *:443 ssl;
+  listen                  443 ssl;
   server_name             git.<span class="userinput">ClusterID.example.com</span>;
   proxy_connect_timeout   90s;
   proxy_read_timeout      300s;
 
-  ssl on;
   ssl_certificate         <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
   ssl_certificate_key     <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
 
diff --git a/doc/install/install-components.html.textile.liquid b/doc/install/install-components.html.textile.liquid
deleted file mode 100644
index 15fbe11622..0000000000
--- a/doc/install/install-components.html.textile.liquid
+++ /dev/null
@@ -1,28 +0,0 @@
----
-layout: default
-navsection: installguide
-title: Choosing which components to install
-...
-
-Arvados consists of many components, some of which may be omitted (at the cost of reduced functionality.)  It may also be helpful to review the "Arvados Architecture":{{site.baseurl}}/architecture to understand how these components interact.
-
-table(table table-bordered table-condensed).
-|\3=. *Core*|
-|"Postgres database":install-postgresql.html |Stores data for the API server.|Required.|
-|"API server":install-api-server.html |Core Arvados logic for managing users, groups, collections, containers, and enforcing permissions.|Required.|
-|\3=. *Keep (storage)*|
-|"Keepstore":install-keepstore.html |Stores content-addressed blocks in a variety of backends (local filesystem, cloud object storage).|Required.|
-|"Keepproxy":install-keepproxy.html |Gateway service to access keep servers from external networks.|Required to be able to use arv-put, arv-get, or arv-mount outside the private Arvados network.|
-|"Keep-web":install-keep-web.html |Gateway service providing read/write HTTP and WebDAV support on top of Keep.|Required to be able to download files from Keep over plain HTTP in Workbench.|
-|"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
-|\3=. *User interface*|
-|"Single Sign On server":install-sso.html |Login server.|Required for web based login to Workbench.|
-|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional.  Depends on API server, SSO server, keep-web, websockets server.|
-|"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional.  Depends on git server (arv-git-httpd).|
-|\3=. *Additional services*|
-|"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
-|"Shell server":install-shell-server.html |Synchronize (create/delete/configure) Unix shell accounts with Arvados users.|Optional.|
-|"Git server":install-arv-git-httpd.html |Arvados-hosted git repositories, with Arvados-token based authentication.|Optional, but required by Workflow Composer.|
-|\3=. *Crunch (running containers)*|
-|"crunch-dispatch-slurm":crunch2-slurm/install-prerequisites.html |Run analysis workflows using Docker containers distributed across a SLURM cluster.|Optional if you wish to use Arvados for data management only.|
-|"Node Manager":install-nodemanager.html |Allocate and free cloud VM instances on demand based on workload.|Optional, not needed for a static SLURM cluster (such as on-premise HPC).|
diff --git a/doc/install/install-keep-web.html.textile.liquid b/doc/install/install-keep-web.html.textile.liquid
index 1ac387b645..b31827bf70 100644
--- a/doc/install/install-keep-web.html.textile.liquid
+++ b/doc/install/install-keep-web.html.textile.liquid
@@ -91,7 +91,7 @@ h2. Set InternalURLs
 <pre><code>    Services:
       WebDAV:
         InternalURLs:
-          http://"<span class="userinput">localhost:9002</span>": {}
+          http://<span class="userinput">localhost:9002</span>: {}
 </code></pre>
 </notextile>
 
@@ -121,7 +121,7 @@ upstream keep-web {
 }
 
 server {
-  listen                *:443 ssl;
+  listen                443 ssl;
   server_name           <span class="userinput">download.ClusterID.example.com</span>
                         <span class="userinput">collections.ClusterID.example.com</span>
                         <span class="userinput">*.collections.ClusterID.example.com</span>
diff --git a/doc/install/install-keepproxy.html.textile.liquid b/doc/install/install-keepproxy.html.textile.liquid
index 0839c0e521..ae6bd3989c 100644
--- a/doc/install/install-keepproxy.html.textile.liquid
+++ b/doc/install/install-keepproxy.html.textile.liquid
@@ -58,7 +58,7 @@ Use a text editor to create a new file @/etc/nginx/conf.d/keepproxy.conf@ with t
 }
 
 server {
-  listen                  *:443 ssl;
+  listen                  443 ssl;
   server_name             <span class="userinput">keep.ClusterID.example.com</span>;
 
   proxy_connect_timeout   90s;
@@ -67,7 +67,6 @@ server {
   proxy_http_version      1.1;
   proxy_request_buffering off;
 
-  ssl on;
   ssl_certificate     <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
   ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
 
diff --git a/doc/install/install-keepstore.html.textile.liquid b/doc/install/install-keepstore.html.textile.liquid
index fedeb3c3f6..869ca15d9e 100644
--- a/doc/install/install-keepstore.html.textile.liquid
+++ b/doc/install/install-keepstore.html.textile.liquid
@@ -61,8 +61,8 @@ Add each keepstore server to the @Services.Keepstore@ section of @/etc/arvados/c
       Keepstore:
         # No ExternalURL because they are only accessed by the internal subnet.
         InternalURLs:
-          "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107/": {}
-          "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107/": {}
+          "http://<span class="userinput">keep0.ClusterID.example.com</span>:25107": {}
+          "http://<span class="userinput">keep1.ClusterID.example.com</span>:25107": {}
           # and so forth
 </code></pre>
 </notextile>
diff --git a/doc/install/install-manual-prerequisites.html.textile.liquid b/doc/install/install-manual-prerequisites.html.textile.liquid
index ea6ad47794..2ce6e36a61 100644
--- a/doc/install/install-manual-prerequisites.html.textile.liquid
+++ b/doc/install/install-manual-prerequisites.html.textile.liquid
@@ -53,8 +53,7 @@ table(table table-bordered table-condensed).
 |"Keep-web":install-keep-web.html |Gateway service providing read/write HTTP and WebDAV support on top of Keep.|Required to access files from Workbench.|
 |"Keep-balance":install-keep-balance.html |Storage cluster maintenance daemon responsible for moving blocks to their optimal server location, adjusting block replication levels, and trashing unreferenced blocks.|Required to free deleted data from underlying storage, and to ensure proper replication and block distribution (including support for storage classes).|
 |\3=. *User interface*|
-|"Single Sign On server":install-sso.html |Web based login to Workbench.|Depends on identity provider.  Not required for Google.  Required for LDAP or standalone database.|
-|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional.  Depends on API server, SSO server, keep-web, websockets server.|
+|"Workbench":install-workbench-app.html, "Workbench2":install-workbench2-app.html |Primary graphical user interface for working with file collections and running containers.|Optional.  Depends on API server, keep-web, websockets server.|
 |"Workflow Composer":install-composer.html |Graphical user interface for editing Common Workflow Language workflows.|Optional.  Depends on git server (arv-git-httpd).|
 |\3=. *Additional services*|
 |"Websockets server":install-ws.html |Event distribution server.|Required to view streaming container logs in Workbench.|
@@ -68,9 +67,9 @@ h2(#identity). Identity provider
 
 Choose which backend you will use to authenticate users.
 
-* Google login to authenticate users with a Google account.  Note: if you only use this identity provider, login can be handled by @arvados-controller@ (recommended), and you do not need to install the Arvados Single Sign-On server (SSO).
-* LDAP login to authenticate users using the LDAP protocol, supported by many services such as OpenLDAP and Active Directory.  Supports username/password authentication.
-* Standalone SSO server user database.  Supports username/password authentication.  Supports new user sign-up.
+* Google login to authenticate users with a Google account.
+* LDAP login to authenticate users by username/password using the LDAP protocol, supported by many services such as OpenLDAP and Active Directory.
+* PAM login to authenticate users by username/password according to the PAM configuration on the controller node.
 
 h2(#storage). Storage backend
 
@@ -102,16 +101,14 @@ For a production installation, this is a reasonable starting point:
 table(table table-bordered table-condensed).
 |_. Function|_. Number of nodes|_. Recommended specs|
 |Postgres database, Arvados API server, Arvados controller, Git, Websockets, Container dispatcher|1|16+ GiB RAM, 4+ cores, fast disk for database|
-|Single Sign-On (SSO) server ^1^|1|2 GiB RAM|
 |Workbench, Keepproxy, Keep-web, Keep-balance|1|8 GiB RAM, 2+ cores|
-|Keepstore servers ^2^|2+|4 GiB RAM|
-|Compute worker nodes ^2^|0+ |Depends on workload; scaled dynamically in the cloud|
-|User shell nodes ^3^|0+|Depends on workload|
+|Keepstore servers ^1^|2+|4 GiB RAM|
+|Compute worker nodes ^1^|0+ |Depends on workload; scaled dynamically in the cloud|
+|User shell nodes ^2^|0+|Depends on workload|
 </div>
 
-^1^ May be omitted when using Google login support in @arvados-controller@
-^2^ Should be scaled up as needed
-^3^ Refers to shell nodes managed by Arvados, that provide ssh access for users to interact with Arvados at the command line.  Optional.
+^1^ Should be scaled up as needed
+^2^ Refers to shell nodes managed by Arvados, that provide ssh access for users to interact with Arvados at the command line.  Optional.
 
 {% include 'notebox_begin' %}
 For a small demo installation, it is possible to run all the Arvados services on a single node.  Special considerations for single-node installs will be noted in boxes like this.
@@ -140,7 +137,6 @@ table(table table-bordered table-condensed).
 |Arvados API|@ClusterID.example.com@|
 |Arvados Git server|git.@ClusterID.example.com@|
 |Arvados Websockets endpoint|ws.@ClusterID.example.com@|
-|Arvados SSO Server|@auth.example.com@|
 |Arvados Workbench|workbench.@ClusterID.example.com@|
 |Arvados Workbench 2|workbench2.@ClusterID.example.com@|
 |Arvados Keepproxy server|keep.@ClusterID.example.com@|
diff --git a/doc/install/install-shell-server.html.textile.liquid b/doc/install/install-shell-server.html.textile.liquid
index 44b3834ab8..5ac5e9e6b8 100644
--- a/doc/install/install-shell-server.html.textile.liquid
+++ b/doc/install/install-shell-server.html.textile.liquid
@@ -69,7 +69,7 @@ As an Arvados admin user (such as the system root user), create a "scoped token"
 
 <notextile>
 <pre>
-<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'
+<code>apiserver:~$ <span class="userinput">arv api_client_authorization create --api-client-authorization '{"scopes":["GET /arvados/v1/virtual_machines/<b>zzzzz-2x53u-zzzzzzzzzzzzzzz</b>/logins"]}'</span>
 {
  ...
  "api_token":"zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz",
diff --git a/doc/install/install-sso.html.textile.liquid b/doc/install/install-sso.html.textile.liquid
deleted file mode 100644
index 4d91b18c00..0000000000
--- a/doc/install/install-sso.html.textile.liquid
+++ /dev/null
@@ -1,237 +0,0 @@
----
-layout: default
-navsection: installguide
-title: Install the Single Sign On (SSO) server
-...
-{% comment %}
-Copyright (C) The Arvados Authors. All rights reserved.
-
-SPDX-License-Identifier: CC-BY-SA-3.0
-{% endcomment %}
-
-{% include 'notebox_begin_warning' %}
-Skip this section if you are using Google login via @arvados-controller@.
-{% include 'notebox_end' %}
-
-# "Install dependencies":#dependencies
-# "Set up database":#database-setup
-# "Update config.yml":#update-config
-# "Configure the SSO server":#create-application-yml
-# "Update Nginx configuration":#update-nginx
-# "Install arvados-sso-server":#install-packages
-# "Create arvados-server client record":#client
-# "Restart the API server and controller":#restart-api
-
-h2(#dependencies). Install dependencies
-
-# "Install PostgreSQL":install-postgresql.html
-# "Install Ruby and Bundler":ruby.html  Important!  The Single Sign On server only supports Ruby 2.3, to avoid version conflicts we recommend installing it on a different server from the API server.  When installing Ruby, ensure that you get the right version by installing the "ruby2.3" package, or by using RVM with @--ruby=2.3@
-# "Install nginx":nginx.html
-# "Install Phusion Passenger":https://www.phusionpassenger.com/library/walkthroughs/deploy/ruby/ownserver/nginx/oss/install_passenger_main.html
-
-h2(#database-setup). Set up the database
-
-{% assign service_role = "arvados_sso" %}
-{% assign service_database = "arvados_sso_production" %}
-{% assign use_contrib = false %}
-{% include 'install_postgres_database' %}
-
-Now create @/etc/arvados/sso/database.yml@
-
-<pre>
-production:
-  adapter: postgresql
-  encoding: utf8
-  database: arvados_sso_production
-  username: arvados_sso
-  password: $password
-  host: localhost
-  template: template0
-</pre>
-
-h2(#update-config). Update config.yml
-
-<pre>
-    Services:
-      SSO:
-        ExternalURL: auth.ClusterID.example.com
-    Login:
-      ProviderAppID: "arvados-server"
-      ProviderAppSecret: $app_secret
-</pre>
-
-Generate @ProviderAppSecret@:
-
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</code></pre></notextile>
-
-h2(#create-application-yml). Configure the SSO server
-
-The SSO server runs from the @/var/www/arvados-sso/current/@ directory. The files @/var/www/arvados-sso/current/config/application.yml@ and @/var/www/arvados-sso/current/config/database.yml@ will be symlinked to the configuration files in @/etc/arvados/sso/@.
-
-The SSO server reads the @config/application.yml@ file, as well as the @config/application.defaults.yml@ file. Values in @config/application.yml@ take precedence over the defaults that are defined in @config/application.defaults.yml@. The @config/application.yml.example@ file is not read by the SSO server and is provided for installation convenience only.
-
-Consult @config/application.default.yml@ for a full list of configuration options.  Local configuration goes in @/etc/arvados/sso/application.yml@, do not edit @config/application.default.yml@.
-
-Create @/etc/arvados/sso/application.yml@ and add these keys:
-
-<pre>
-production:
-  uuid_prefix: xxxxx
-  secret_token: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</pre>
-
-h3(#uuid_prefix). uuid_prefix
-
-Most of the time, you want this to be the same as your @ClusterID@.  If not, generate a new one from the command line listed previously.
-
-h3(#secret_token). secret_token
-
-Generate a new secret token for signing cookies:
-
-<notextile>
-<pre><code>~$ <span class="userinput">ruby -e 'puts rand(2**400).to_s(36)'</span>
-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
-</code></pre></notextile>
-
-h3(#authentication_methods). Authentication methods
-
-Authentication methods are configured in @application.yml@.  Currently three authentication methods are supported: local accounts, LDAP, and Google.  If neither Google nor LDAP are enabled, the SSO server defaults to local user accounts.   Only one authentication mechanism should be in use at a time.  Choose your authentication method and add the listed configuration items to the @production@ section.
-
-h4(#local_accounts). Local account authentication
-
-There are two configuration options for local accounts:
-
-<pre>
-  # If true, allow new creation of new accounts in the SSO server's internal
-  # user database.
-  allow_account_registration: false
-
-  # If true, send an email confirmation before activating new accounts in the
-  # SSO server's internal user database (otherwise users are activated immediately.)
-  require_email_confirmation: false
-</pre>
-
-For more information about configuring backend support for sending email (required to send email confirmations) see "Configuring Action Mailer":http://guides.rubyonrails.org/configuring.html#configuring-action-mailer
-
-If @allow_account_registration@ is false, you may manually create local accounts on the SSO server from the Rails console.  {% include 'install_rails_command' %}
-
-Enter the following commands at the console.
-
-<notextile>
-<pre><code>:001 &gt; <span class="userinput">user = User.new(:email =&gt; "test@example.com")</span>
-:002 &gt; <span class="userinput">user.password = "passw0rd"</span>
-:003 &gt; <span class="userinput">user.save!</span>
-:004 &gt; <span class="userinput">quit</span>
-</code></pre>
-</notextile>
-
-h4(#ldap). LDAP authentication
-
-The following options are available to configure LDAP authentication.  Note that you must preserve the indentation of the fields listed under @use_ldap@.
-
-<pre>
-  use_ldap:
-    title: Example LDAP
-    host: ldap.example.com
-    port: 636
-    method: ssl
-    base: "ou=Users, dc=example, dc=com"
-    uid: uid
-    email_domain: example.com
-    #bind_dn: "some_user"
-    #password: "some_password"
-</pre>
-
-table(table).
-|_. Option|_. Description|
-|title |Title displayed to the user on the login page|
-|host  |LDAP server hostname|
-|port  |LDAP server port|
-|method|One of "plain", "ssl", "tls"|
-|base  |Directory lookup base|
-|uid   |User id field used for directory lookup|
-|email_domain|Strip off specified email domain from login and perform lookup on bare username|
-|bind_dn|If required by server, username to log with in before performing directory lookup|
-|password|If required by server, password to log with before performing directory lookup|
-
-h4(#google). Google authentication
-
-First, visit "Setting up Google auth.":google-auth.html
-
-Next, copy the values of *Client ID* and *Client secret* from the Google Developers Console into the Google section of @config/application.yml@, like this:
-
-<notextile>
-<pre><code>  # Google API tokens required for OAuth2 login.
-  google_oauth2_client_id: <span class="userinput">"---YOUR---CLIENT---ID---HERE--"-</span>
-  google_oauth2_client_secret: <span class="userinput">"---YOUR---CLIENT---SECRET---HERE--"-</span></code></pre></notextile>
-
-h2(#update-nginx). Update nginx configuration
-
-Use a text editor to create a new file @/etc/nginx/conf.d/arvados-sso.conf@ with the following configuration.  Options that need attention are marked in <span class="userinput">red</span>.
-
-<notextile>
-<pre><code>server {
-  listen       <span class="userinput">auth.ClusterID.example.com</span>:443 ssl;
-  server_name  <span class="userinput">auth.ClusterID.example.com</span>;
-
-  ssl on;
-  ssl_certificate     <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
-  ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
-
-  root   /var/www/arvados-sso/current/public;
-  index  index.html;
-
-  passenger_enabled on;
-
-  # <span class="userinput">If you are using RVM, uncomment the line below.</span>
-  # <span class="userinput">If you're using system ruby, leave it commented out.</span>
-  #passenger_ruby /usr/local/rvm/wrappers/default/ruby;
-}
-</code></pre>
-</notextile>
-
-h2(#install-packages). Install arvados-sso-server package
-
-h3. Centos 7
-
-<notextile>
-<pre><code># <span class="userinput">yum install arvados-sso-server</span>
-</code></pre>
-</notextile>
-
-h3. Debian and Ubuntu
-
-<notextile>
-<pre><code># <span class="userinput">apt-get --no-install-recommends arvados-sso-server</span>
-</code></pre>
-</notextile>
-
-h2(#client). Create arvados-server client record
-
-{% assign railshost = "" %}
-{% assign railsdir = "/var/www/arvados-sso/current" %}
-Use @rails console@ to create a @Client@ record that will be used by the Arvados API server.  {% include 'install_rails_command' %}
-
-Enter the following commands at the console.  The values that appear after you assign @app_id@ and @app_secret@ will be copied to  @Login.ProviderAppID@ and @Login.ProviderAppSecret@ in @config.yml@.
-
-<notextile>
-<pre><code>:001 &gt; <span class="userinput">c = Client.new</span>
-:002 &gt; <span class="userinput">c.name = "joshid"</span>
-:003 &gt; <span class="userinput">c.app_id = "arvados-server"</span>
-:004 &gt; <span class="userinput">c.app_secret = "the value of Login.ProviderAppSecret"</span>
-:005 &gt; <span class="userinput">c.save!</span>
-:006 &gt; <span class="userinput">quit</span>
-</code></pre>
-</notextile>
-
-h2(#restart-api). Restart the API server and controller
-
-After adding the SSO server to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
-
-<notextile>
-<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
-</code></pre>
-</notextile>
diff --git a/doc/install/install-webshell.html.textile.liquid b/doc/install/install-webshell.html.textile.liquid
new file mode 100644
index 0000000000..ae6a8d2109
--- /dev/null
+++ b/doc/install/install-webshell.html.textile.liquid
@@ -0,0 +1,183 @@
+---
+layout: default
+navsection: installguide
+title: Configure webshell
+...
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+# "Introduction":#introduction
+# "Prerequisites":#prerequisites
+# "Update config.yml":#configure
+# "Update nginx configuration":#update-nginx
+# "Install packages":#install-packages
+# "Configure shellinabox":#config-shellinabox
+# "Configure pam":#config-pam
+# "Confirm working installation":#confirm-working
+
+h2(#introduction). Introduction
+
+Arvados supports @webshell@, which allows ssh access to shell nodes via the browser. This functionality is integrated in @Workbench@.
+
+@Webshell@ is provided by the @shellinabox@ package which runs on each shell node for which webshell is enabled. For authentication, a supported @pam library@ that allows authentication against Arvados is also required. One Nginx (or similar web server) virtualhost is also needed to expose all the @shellinabox@ instances via https.
+
+h2(#prerequisites). Prerequisites
+
+# "Install workbench":{{site.baseurl}}/install/install-workbench-app.html
+# "Set up a shell node":{{site.baseurl}}/install/install-shell-server.html
+
+h2(#configure). Update config.yml
+
+Edit the cluster config at @config.yml@ and set @Services.WebShell.ExternalURL@.  Replace @zzzzz@ with your cluster id. Workbench will use this information to activate its support for webshell.
+
+<notextile>
+<pre><code>    Services:
+      WebShell:
+        InternalURLs: {}
+        ExternalURL: <span class="userinput">https://webshell.ClusterID.example.com/</span>
+</span></code></pre>
+</notextile>
+
+h2(#update-nginx). Update Nginx configuration
+
+The arvados-webshell service will be accessible from anywhere on the internet, so we recommend using SSL for transport encryption. This Nginx virtualhost could live on your Workbench server, or any other server that is reachable by your Workbench users and can access the @shell-in-a-box@ service on the shell node(s) on port 4200.
+
+Use a text editor to create a new file @/etc/nginx/conf.d/arvados-webshell.conf@ with the following configuration.  Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile><pre>
+upstream arvados-webshell {
+  server                <span class="userinput">shell.ClusterID.example.com</span>:<span class="userinput">4200</span>;
+}
+
+server {
+  listen                443 ssl;
+  server_name           webshell.<span class="userinput">ClusterID.example.com</span>;
+
+  proxy_connect_timeout 90s;
+  proxy_read_timeout    300s;
+
+  ssl                   on;
+  ssl_certificate       <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
+  ssl_certificate_key   <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
+
+  location /<span class="userinput">shell.ClusterID</span> {
+    if ($request_method = 'OPTIONS') {
+       add_header 'Access-Control-Allow-Origin' '*'; 
+       add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+       add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+       add_header 'Access-Control-Max-Age' 1728000;
+       add_header 'Content-Type' 'text/plain charset=UTF-8';
+       add_header 'Content-Length' 0;
+       return 204;
+    }
+    if ($request_method = 'POST') {
+       add_header 'Access-Control-Allow-Origin' '*';
+       add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+       add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+    }
+    if ($request_method = 'GET') {
+       add_header 'Access-Control-Allow-Origin' '*';
+       add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS';
+       add_header 'Access-Control-Allow-Headers' 'DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type';
+    }
+
+    proxy_ssl_session_reuse off;
+    proxy_read_timeout  90;
+    proxy_set_header    X-Forwarded-Proto https;
+    proxy_set_header    Host $http_host;
+    proxy_set_header    X-Real-IP $remote_addr;
+    proxy_set_header    X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_pass          http://arvados-webshell;
+  }
+}
+</pre></notextile>
+
+Note that the location line in the nginx config matches your shell node hostname *without domain*, because that is how the shell node was defined in the "Set up a shell node":{{site.baseurl}}/install/install-shell-server.html#vm-record instructions. It makes for a more user friendly experience in Workbench.
+
+For additional shell nodes with @shell-in-a-box@, add @location@ and @upstream@ sections as needed.
+
+{% assign arvados_component = 'shellinabox libpam-arvados-go' %}
+
+{% include 'install_packages' %}
+
+h2(#config-shellinabox). Configure shellinabox
+
+h3. Red Hat and Centos
+
+Edit @/etc/sysconfig/shellinaboxd@:
+
+<notextile><pre>
+# TCP port that shellinboxd's webserver listens on
+PORT=4200
+
+# SSL is disabled because it is terminated in Nginx. Adjust as needed.
+OPTS="--disable-ssl --no-beep --service=/<span class="userinput">shell.ClusterID.example.com</span>:AUTH:HOME:SHELL"
+</pre></notextile>
+
+<notextile>
+<pre>
+<code># <span class="userinput">systemctl enable shellinabox</span></code>
+<code># <span class="userinput">systemctl start shellinabox</span></code>
+</pre>
+</notextile>
+
+h3. Debian and Ubuntu
+
+Edit @/etc/default/shellinabox@:
+
+<notextile><pre>
+# TCP port that shellinboxd's webserver listens on
+SHELLINABOX_PORT=4200
+
+# SSL is disabled because it is terminated in Nginx. Adjust as needed.
+SHELLINABOX_ARGS="--disable-ssl --no-beep --service=/<span class="userinput">shell.ClusterID.example.com</span>:AUTH:HOME:SHELL"
+</pre></notextile>
+
+<notextile>
+<pre>
+<code># <span class="userinput">systemctl enable shellinabox</span></code>
+<code># <span class="userinput">systemctl start shellinabox</span></code>
+</pre>
+</notextile>
+
+
+h2(#config-pam). Configure pam
+
+Use a text editor to create a new file @/etc/pam.d/shellinabox@ with the following configuration. Options that need attention are marked in <span class="userinput">red</span>.
+
+<notextile><pre>
+# This example is a stock debian "login" file with pam_arvados
+# replacing pam_unix. It can be installed as /etc/pam.d/shellinabox .
+
+auth       optional   pam_faildelay.so  delay=3000000
+auth [success=ok new_authtok_reqd=ok ignore=ignore user_unknown=bad default=die] pam_securetty.so
+auth       requisite  pam_nologin.so
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so close
+session       required   pam_env.so readenv=1
+session       required   pam_env.so readenv=1 envfile=/etc/default/locale
+
+auth [success=1 default=ignore] /usr/lib/pam_arvados.so <span class="userinput">ClusterID.example.com</span> <span class="userinput">shell.ClusterID.example.com</span>
+auth    requisite            pam_deny.so
+auth    required            pam_permit.so
+
+auth       optional   pam_group.so
+session    required   pam_limits.so
+session    optional   pam_lastlog.so
+session    optional   pam_motd.so  motd=/run/motd.dynamic
+session    optional   pam_motd.so
+session    optional   pam_mail.so standard
+
+@include common-account
+@include common-session
+@include common-password
+
+session [success=ok ignore=ignore module_unknown=ignore default=bad] pam_selinux.so open
+</pre></notextile>
+
+h2(#confirm-working). Confirm working installation
+
+A user should be able to log in to the shell server, using webshell via workbench. Please refer to "Accessing an Arvados VM with Webshell":{{site.baseurl}}/user/getting_started/vm-login-with-webshell.html
+
diff --git a/doc/install/install-workbench-app.html.textile.liquid b/doc/install/install-workbench-app.html.textile.liquid
index 3d391724dc..7ee8db92f1 100644
--- a/doc/install/install-workbench-app.html.textile.liquid
+++ b/doc/install/install-workbench-app.html.textile.liquid
@@ -62,10 +62,9 @@ Use a text editor to create a new file @/etc/nginx/conf.d/arvados-workbench.conf
 }
 
 server {
-  listen       *:443 ssl;
+  listen       443 ssl;
   server_name  workbench.<span class="userinput">ClusterID.example.com</span>;
 
-  ssl on;
   ssl_certificate     <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
   ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
 
diff --git a/doc/install/install-workbench2-app.html.textile.liquid b/doc/install/install-workbench2-app.html.textile.liquid
index b59799c43f..f3a320b102 100644
--- a/doc/install/install-workbench2-app.html.textile.liquid
+++ b/doc/install/install-workbench2-app.html.textile.liquid
@@ -47,10 +47,9 @@ Use a text editor to create a new file @/etc/nginx/conf.d/arvados-workbench2.con
 }
 
 server {
-  listen       *:443 ssl;
+  listen       443 ssl;
   server_name  workbench2.<span class="userinput">ClusterID.example.com</span>;
 
-  ssl on;
   ssl_certificate     <span class="userinput">/YOUR/PATH/TO/cert.pem</span>;
   ssl_certificate_key <span class="userinput">/YOUR/PATH/TO/cert.key</span>;
 
diff --git a/doc/install/install-ws.html.textile.liquid b/doc/install/install-ws.html.textile.liquid
index e7b20f45a0..2b982504f2 100644
--- a/doc/install/install-ws.html.textile.liquid
+++ b/doc/install/install-ws.html.textile.liquid
@@ -43,7 +43,7 @@ upstream arvados-ws {
 }
 
 server {
-  listen                *:443 ssl;
+  listen                443 ssl;
   server_name           ws.<span class="userinput">ClusterID.example.com</span>;
 
   proxy_connect_timeout 90s;
@@ -71,21 +71,12 @@ server {
 
 {% include 'restart_api' %}
 
-h2(#restart-api). Restart the API server and controller
-
-After adding the SSO server to the Services section, make sure the cluster config file is up to date on the API server host, and restart the API server and controller processes to ensure the changes are applied.
-
-<notextile>
-<pre><code># <span class="userinput">systemctl restart nginx arvados-controller</span>
-</code></pre>
-</notextile>
-
 h2(#confirm). Confirm working installation
 
 Confirm the service is listening on its assigned port and responding to requests.
 
 <notextile>
-<pre><code>~$ <span class="userinput">curl https://<span class="userinput">ws.ClusterID.example.com</span>/status.json</span>
-{"Clients":1}
+<pre><code>~$ <span class="userinput">curl https://<span class="userinput">ws.ClusterID.example.com</span>/websocket</span>
+not websocket protocol
 </code></pre>
 </notextile>
diff --git a/doc/install/setup-login.html.textile.liquid b/doc/install/setup-login.html.textile.liquid
index b88ba49984..aec82cfe2a 100644
--- a/doc/install/setup-login.html.textile.liquid
+++ b/doc/install/setup-login.html.textile.liquid
@@ -9,21 +9,97 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-# "Option 1: Google login through Arvados controller":#controller
-# "Option 2: Separate single-sign-on (SSO) server (Google, LDAP, local database)":#sso
+Select one of the following login mechanisms for your cluster.
 
-h2(#controller). Option 1: Google login through Arvados controller
+# If all users will authenticate with Google, "configure Google login":#google.
+# If all users will authenticate with an OpenID Connect provider (other than Google), "configure OpenID Connect":#oidc.
+# If all users will authenticate with an existing LDAP service, "configure LDAP":#ldap.
+# If all users will authenticate using PAM as configured on your controller node, "configure PAM":#pam.
 
-First, visit "Setting up Google auth.":google-auth.html
+h2(#google). Google login
 
-Next, copy the values of *Client ID* and *Client secret* from the Google Developers Console into @Login.GoogleClientID@ and @Login.GoogleClientSecret@ of @config.yml@ :
+With this configuration, users will sign in with their Google accounts.
+
+Use the <a href="https://console.developers.google.com" target="_blank">Google Developers Console</a> to create a set of client credentials.
+# Select or create a project.
+# Click *+ Enable APIs and Services*.
+#* Search for *People API* and click *Enable API*.
+#* Navigate back to the main "APIs & Services" page.
+# On the sidebar, click *OAuth consent screen*.
+#* On consent screen settings, enter your identifying details.
+#* Under *Authorized domains* add your domain (@example.com@).
+#* Click *Save*.
+# On the sidebar, click *Credentials*, then click *Create credentials*&rarr;*OAuth client ID*
+# Under *Application type* select *Web application*.
+# Add the JavaScript origin: @https://ClusterID.example.com/@
+# Add the Redirect URI: @https://ClusterID.example.com/login@
+# Copy the values of *Client ID* and *Client secret* to the @Login.Google@ section of @config.yml@.
+
+<pre>
+    Login:
+      Google:
+        Enable: true
+        ClientID: "0000000000000-zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz.apps.googleusercontent.com"
+        ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
+</pre>
+
+h2(#oidc). OpenID Connect
+
+With this configuration, users will sign in with a third-party OpenID Connect provider. The provider will supply appropriate values for the issuer URL, client ID, and client secret config entries.
 
 <pre>
     Login:
-      GoogleClientID: ""
-      GoogleClientSecret: ""
+      OpenIDConnect:
+        Enable: true
+        Issuer: https://accounts.example.com/
+        ClientID: "0123456789abcdef"
+        ClientSecret: "zzzzzzzzzzzzzzzzzzzzzzzz"
 </pre>
 
-h2(#sso). Option 2: Separate single-sign-on (SSO) server (supports Google, LDAP, local database)
+Check the OpenIDConnect section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
+
+h2(#ldap). LDAP
+
+With this configuration, authentication uses an external LDAP service like OpenLDAP or Active Directory.
+
+Enable LDAP authentication and provide your LDAP server's host, port, and credentials (if needed to search the directory) in @config.yml@:
+
+<pre>
+    Login:
+      LDAP:
+        Enable: true
+        URL: ldap://ldap.example.com:389
+        SearchBindUser: cn=lookupuser,dc=example,dc=com
+        SearchBindPassword: xxxxxxxx
+        SearchBase: ou=Users,dc=example,dc=com
+</pre>
+
+The email address reported by LDAP will be used as primary key for Arvados accounts. This means *users must not be able to edit their own email addresses* in the directory.
+
+Additional configuration settings are available:
+* @StartTLS@ is enabled by default.
+* @StripDomain@ and @AppendDomain@ modify the username entered by the user before searching for it in the directory.
+* @SearchAttribute@ (default @uid@) is the LDAP attribute used when searching for usernames.
+* @SearchFilters@ accepts LDAP filter expressions to control which users can log in.
+
+Check the LDAP section in the "default config file":{{site.baseurl}}/admin/config.html for more details and configuration options.
+
+h2(#pam). PAM (experimental)
+
+With this configuration, authentication is done according to the Linux PAM ("Pluggable Authentication Modules") configuration on your controller host.
+
+Enable PAM authentication in @config.yml@:
+
+<pre>
+    Login:
+      PAM:
+        Enable: true
+</pre>
+
+Check the "default config file":{{site.baseurl}}/admin/config.html for more PAM configuration options.
+
+The default PAM configuration on most Linux systems uses the local password database in @/etc/shadow@ for all logins. In this case, in order to log in to Arvados, users must have a shell account and password on the controller host itself. This can be convenient for a single-user or test cluster.
+
+PAM can also be configured to use different backends like LDAP. In a production environment, PAM configuration should use the service name ("arvados" by default) to set a separate policy for Arvados logins: generally, Arvados users should not have shell accounts on the controller node.
 
-See "Install the Single Sign On (SSO) server":install-sso.html
+For information about configuring PAM, refer to the "PAM System Administrator's Guide":http://www.linux-pam.org/Linux-PAM-html/Linux-PAM_SAG.html.
diff --git a/doc/user/getting_started/community.html.textile.liquid b/doc/user/getting_started/community.html.textile.liquid
index b85b556123..6dc0b4d816 100644
--- a/doc/user/getting_started/community.html.textile.liquid
+++ b/doc/user/getting_started/community.html.textile.liquid
@@ -11,15 +11,19 @@ SPDX-License-Identifier: CC-BY-SA-3.0
 
 h2. On the web
 
-The Arvados Free Sofware project page is located at "https://arvados.org":https://arvados.org .  The "Arvados Wiki":https://dev.arvados.org/projects/arvados/wiki is a collaborative site for documenting Arvados and provides an overview of the Arvados Platform and Components.  The "Arvados blog":https://dev.arvados.org/projects/arvados/blogs posts articles of interest about Arvados.
+The Arvados Free Sofware project page is located at "https://arvados.org":https://arvados.org .  The "Arvados Wiki":https://dev.arvados.org/projects/arvados/wiki is a collaborative site for documenting Arvados and provides an overview of the Arvados Platform and Components.
 
-h2. Mailing lists
+h2. Forum
 
-The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvados is a forum for general discussion, questions, and news about Arvados development.  The "Arvados developer mailing list":http://lists.arvados.org/mailman/listinfo/arvados-dev is a forum for more technical discussion, intended for developers and contributers to Arvados.
+The "Arvados Forum":https://forum.arvados.org has topic-based discussion, Q&A and community support.
 
 h2. Chat
 
-The "arvados community channel":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
+The "arvados/community":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
+
+h2. Mailing list
+
+The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvados is a low-volume list used mainly to announce new releases of Arvados.
 
 h2. Bug tracking
 
diff --git a/doc/user/reference/api-tokens.html.textile.liquid b/doc/user/reference/api-tokens.html.textile.liquid
index d5172f0c5b..6afc20bf4f 100644
--- a/doc/user/reference/api-tokens.html.textile.liquid
+++ b/doc/user/reference/api-tokens.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
 SPDX-License-Identifier: CC-BY-SA-3.0
 {% endcomment %}
 
-The Arvados API token is a secret key that enables the @arv@ command line client to access Arvados with the proper permissions.
+The Arvados API token is a secret key that enables the Arvados command line tools to authenticate themselves.
 
 Access the Arvados Workbench using this link: "{{site.arvados_workbench_host}}/":{{site.arvados_workbench_host}}/  (Replace the hostname portion with the hostname of your local Arvados instance if necessary.)
 
diff --git a/doc/user/topics/arvados-sync-groups.html.textile.liquid b/doc/user/topics/arvados-sync-groups.html.textile.liquid
index 9a609039b4..7d831bf040 100644
--- a/doc/user/topics/arvados-sync-groups.html.textile.liquid
+++ b/doc/user/topics/arvados-sync-groups.html.textile.liquid
@@ -15,10 +15,12 @@ h1. Using arvados-sync-groups
 
 This tool reads a CSV (comma-separated values) file having information about external groups and their members. When running it for the first time, it'll create a special group named 'Externally synchronized groups' meant to be the parent of all the remote groups.
 
-Every line on the file should have 2 values: a group name and a local user identifier, meaning that the named user is a member of the group. The tool will create the group if it doesn't exist, and add the user to it. If group member is not present on the input file, the account will be removed from the group.
+Every line on the file should have 3 values: a group name, a local user identifier and a permission level, meaning that the named user is a member of the group with the provided permission. The tool will create the group if it doesn't exist, and add the user to it. If any group member is not present on the input file, it will be removed from the group.
 
 Users can be identified by their email address or username: the tool will check if every user exist on the system, and report back when not found. Groups on the other hand, are identified by their name.
 
+Permission level can be one of the following: @can_read@, @can_write@ or @can_manage@, giving the group member read, read/write or managing privileges on the group. For backwards compatibility purposes, if any record omits the third (permission) field, it will default to @can_write@ permission. You can read more about permissions on the "group management admin guide":/admin/group-management.html.
+
 This tool is designed to be run periodically reading a file created by a remote auth system (ie: LDAP) dump script, applying what's included on the file as the source of truth.
 
 
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 876ac4f9f4..15993c4bc3 100644
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -3,8 +3,8 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Based on Debian Stretch
-FROM debian:stretch-slim
-MAINTAINER Peter Amstutz <peter.amstutz@curii.com>
+FROM debian:buster-slim
+MAINTAINER Arvados Package Maintainers <packaging@arvados.org>
 
 ENV DEBIAN_FRONTEND noninteractive
 
diff --git a/docker/jobs/apt.arvados.org-dev.list b/docker/jobs/apt.arvados.org-dev.list
index 468000ed29..4de87397bc 100644
--- a/docker/jobs/apt.arvados.org-dev.list
+++ b/docker/jobs/apt.arvados.org-dev.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch-dev main
+deb http://apt.arvados.org/ buster-dev main
diff --git a/docker/jobs/apt.arvados.org-stable.list b/docker/jobs/apt.arvados.org-stable.list
index afbc51effe..7882afd01c 100644
--- a/docker/jobs/apt.arvados.org-stable.list
+++ b/docker/jobs/apt.arvados.org-stable.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch main
+deb http://apt.arvados.org/ buster main
diff --git a/docker/jobs/apt.arvados.org-testing.list b/docker/jobs/apt.arvados.org-testing.list
index c8ea91d070..3bb599087e 100644
--- a/docker/jobs/apt.arvados.org-testing.list
+++ b/docker/jobs/apt.arvados.org-testing.list
@@ -1,2 +1,2 @@
 # apt.arvados.org
-deb http://apt.arvados.org/ stretch-testing main
+deb http://apt.arvados.org/ buster-testing main
diff --git a/go.mod b/go.mod
index 4491b35981..884d1fcdac 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,8 @@ require (
 	github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect
 	github.com/arvados/cgofuse v1.2.0-arvados1
 	github.com/aws/aws-sdk-go v1.25.30
+	github.com/bgentry/speakeasy v0.1.0 // indirect
+	github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/coreos/go-systemd v0.0.0-20180108085132-cc4f39464dc7
 	github.com/dgrijalva/jwt-go v3.1.0+incompatible // indirect
@@ -20,9 +22,13 @@ require (
 	github.com/docker/docker v1.4.2-0.20180109013817-94b8a116fbf1
 	github.com/docker/go-connections v0.3.0 // indirect
 	github.com/docker/go-units v0.3.3-0.20171221200356-d59758554a3d // indirect
+	github.com/dustin/go-humanize v1.0.0
 	github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 // indirect
+	github.com/fsnotify/fsnotify v1.4.9
 	github.com/ghodss/yaml v1.0.0
 	github.com/gliderlabs/ssh v0.2.2 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.4.1 // indirect
+	github.com/go-ldap/ldap v3.0.3+incompatible
 	github.com/gogo/protobuf v1.1.1
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/mux v1.6.1-0.20180107155708-5bbbb5b2b572
@@ -30,6 +36,7 @@ require (
 	github.com/imdario/mergo v0.3.8-0.20190415133143-5ef87b449ca7
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff
+	github.com/jmoiron/sqlx v1.2.0
 	github.com/julienschmidt/httprouter v1.2.0
 	github.com/karalabe/xgo v0.0.0-20191115072854-c5ccff8648a7 // indirect
 	github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5 // indirect
@@ -55,6 +62,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45
 	golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd
 	google.golang.org/api v0.13.0
+	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405
 	gopkg.in/square/go-jose.v2 v2.3.1
 	gopkg.in/src-d/go-billy.v4 v4.0.1
diff --git a/go.sum b/go.sum
index 18cf89b0e1..ead655c9b2 100644
--- a/go.sum
+++ b/go.sum
@@ -27,6 +27,10 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092 h1:0Di2onNnlN5PAyWPbqlPyN45eOQ+QW/J9eqLynt4IV4=
+github.com/bradleypeabody/godap v0.0.0-20170216002349-c249933bc092/go.mod h1:8IzBjZCRSnsvM6MJMG8HNNtnzMl48H22rbJL2kRUJ0Y=
 github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
 github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
@@ -52,16 +56,25 @@ github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF
 github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.3.3-0.20171221200356-d59758554a3d h1:dVaNRYvaGV23AdNdsm+4y1mPN0tj3/1v6taqKMmM6Ko=
 github.com/docker/go-units v0.3.3-0.20171221200356-d59758554a3d/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=
 github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
+github.com/go-asn1-ber/asn1-ber v1.4.1 h1:qP/QDxOtmMoJVgXHCXNzDpA0+wkgYB2x5QoLMVOciyw=
+github.com/go-asn1-ber/asn1-ber v1.4.1/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-ldap/ldap v3.0.3+incompatible h1:HTeSZO8hWMS1Rgb2Ziku6b8a7qRIZZMHjsvuZyatzwk=
+github.com/go-ldap/ldap v3.0.3+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@@ -99,6 +112,8 @@ github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff h1:6NvhExg4omUC9
 github.com/jmcvetta/randutil v0.0.0-20150817122601-2bb1b664bcff/go.mod h1:ddfPX8Z28YMjiqoaJhNBzWHapTHXejnB5cDCUWDwriw=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
@@ -111,10 +126,12 @@ github.com/kevinburke/ssh_config v0.0.0-20171013211458-802051befeb5/go.mod h1:CT
 github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU=
 github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c h1:ouxemItv3B/Zh008HJkEXDYCN3BIRyNHxtUN7ThJ5Js=
 github.com/marstr/guid v1.1.1-0.20170427235115-8bdf7d1a087c/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v0.0.0-20161203194507-b8bc1bf76747 h1:eQox4Rh4ewJF+mqYPxCkmBAirRnPaHEB26UkNuPyjlk=
@@ -213,6 +230,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd h1:3x5uuvBgE6oaXJjCOvpCC1IpgJogqQ+PqGGU3ZxAgII=
 golang.org/x/sys v0.0.0-20191105231009-c1f44814a5cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -244,6 +262,8 @@ google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZi
 google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM=
+gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405 h1:829vOVxxusYHC+IqBtkX5mbKtsY9fheQiQn0MZRVLfQ=
 gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
diff --git a/lib/boot/cmd.go b/lib/boot/cmd.go
index 5147e3ac33..e0e2755220 100644
--- a/lib/boot/cmd.go
+++ b/lib/boot/cmd.go
@@ -29,24 +29,33 @@ type supervisedTask interface {
 	String() string
 }
 
+var errNeedConfigReload = errors.New("config changed, restart needed")
+
 type bootCommand struct{}
 
-func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
-	super := &Supervisor{
-		Stderr: stderr,
-		logger: ctxlog.New(stderr, "json", "info"),
+func (bcmd bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	logger := ctxlog.New(stderr, "json", "info")
+	ctx := ctxlog.Context(context.Background(), logger)
+	for {
+		err := bcmd.run(ctx, prog, args, stdin, stdout, stderr)
+		if err == errNeedConfigReload {
+			continue
+		} else if err != nil {
+			logger.WithError(err).Info("exiting")
+			return 1
+		} else {
+			return 0
+		}
 	}
+}
 
-	ctx := ctxlog.Context(context.Background(), super.logger)
+func (bcmd bootCommand) run(ctx context.Context, prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-
-	var err error
-	defer func() {
-		if err != nil {
-			super.logger.WithError(err).Info("exiting")
-		}
-	}()
+	super := &Supervisor{
+		Stderr: stderr,
+		logger: ctxlog.FromContext(ctx),
+	}
 
 	flags := flag.NewFlagSet(prog, flag.ContinueOnError)
 	flags.SetOutput(stderr)
@@ -60,26 +69,25 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
 	flags.BoolVar(&super.OwnTemporaryDatabase, "own-temporary-database", false, "bring up a postgres server and create a temporary database")
 	timeout := flags.Duration("timeout", 0, "maximum time to wait for cluster to be ready")
 	shutdown := flags.Bool("shutdown", false, "shut down when the cluster becomes ready")
-	err = flags.Parse(args)
+	err := flags.Parse(args)
 	if err == flag.ErrHelp {
-		err = nil
-		return 0
+		return nil
 	} else if err != nil {
-		return 2
+		return err
 	} else if *versionFlag {
-		return cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+		cmd.Version.RunCommand(prog, args, stdin, stdout, stderr)
+		return nil
 	} else if super.ClusterType != "development" && super.ClusterType != "test" && super.ClusterType != "production" {
-		err = fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
-		return 2
+		return fmt.Errorf("cluster type must be 'development', 'test', or 'production'")
 	}
 
 	loader.SkipAPICalls = true
 	cfg, err := loader.Load()
 	if err != nil {
-		return 1
+		return err
 	}
 
-	super.Start(ctx, cfg)
+	super.Start(ctx, cfg, loader.Path)
 	defer super.Stop()
 
 	var timer *time.Timer
@@ -89,20 +97,19 @@ func (bootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdou
 
 	url, ok := super.WaitReady()
 	if timer != nil && !timer.Stop() {
-		err = errors.New("boot timed out")
-		return 1
+		return errors.New("boot timed out")
 	} else if !ok {
-		err = errors.New("boot failed")
-		return 1
-	}
-	// Write controller URL to stdout. Nothing else goes to
-	// stdout, so this provides an easy way for a calling script
-	// to discover the controller URL when everything is ready.
-	fmt.Fprintln(stdout, url)
-	if *shutdown {
-		super.Stop()
+		super.logger.Error("boot failed")
+	} else {
+		// Write controller URL to stdout. Nothing else goes
+		// to stdout, so this provides an easy way for a
+		// calling script to discover the controller URL when
+		// everything is ready.
+		fmt.Fprintln(stdout, url)
+		if *shutdown {
+			super.Stop()
+		}
 	}
 	// Wait for signal/crash + orderly shutdown
-	<-super.done
-	return 0
+	return super.Wait()
 }
diff --git a/lib/boot/nginx.go b/lib/boot/nginx.go
index ecbb7a9d3a..0f105d6b6c 100644
--- a/lib/boot/nginx.go
+++ b/lib/boot/nginx.go
@@ -47,6 +47,7 @@ func (runNginx) Run(ctx context.Context, fail func(error), super *Supervisor) er
 		{"KEEPWEBDL", super.cluster.Services.WebDAVDownload},
 		{"KEEPPROXY", super.cluster.Services.Keepproxy},
 		{"GIT", super.cluster.Services.GitHTTP},
+		{"HEALTH", super.cluster.Services.Health},
 		{"WORKBENCH1", super.cluster.Services.Workbench1},
 		{"WS", super.cluster.Services.Websocket},
 	} {
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index 7f5d6a9baa..e38a4775e8 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -19,15 +19,18 @@ import (
 	"os/signal"
 	"os/user"
 	"path/filepath"
+	"reflect"
 	"strings"
 	"sync"
 	"syscall"
 	"time"
 
+	"git.arvados.org/arvados.git/lib/config"
 	"git.arvados.org/arvados.git/lib/service"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"git.arvados.org/arvados.git/sdk/go/health"
+	"github.com/fsnotify/fsnotify"
 	"github.com/sirupsen/logrus"
 )
 
@@ -45,7 +48,8 @@ type Supervisor struct {
 
 	ctx           context.Context
 	cancel        context.CancelFunc
-	done          chan struct{}
+	done          chan struct{} // closed when child procs/services have shut down
+	err           error         // error that caused shutdown (valid when done is closed)
 	healthChecker *health.Aggregator
 	tasksReady    map[string]chan bool
 	waitShutdown  sync.WaitGroup
@@ -55,30 +59,66 @@ type Supervisor struct {
 	environ    []string // for child processes
 }
 
-func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config) {
+func (super *Supervisor) Start(ctx context.Context, cfg *arvados.Config, cfgPath string) {
 	super.ctx, super.cancel = context.WithCancel(ctx)
 	super.done = make(chan struct{})
 
 	go func() {
+		defer close(super.done)
+
 		sigch := make(chan os.Signal)
 		signal.Notify(sigch, syscall.SIGINT, syscall.SIGTERM)
 		defer signal.Stop(sigch)
 		go func() {
 			for sig := range sigch {
 				super.logger.WithField("signal", sig).Info("caught signal")
+				if super.err == nil {
+					super.err = fmt.Errorf("caught signal %s", sig)
+				}
+				super.cancel()
+			}
+		}()
+
+		hupch := make(chan os.Signal)
+		signal.Notify(hupch, syscall.SIGHUP)
+		defer signal.Stop(hupch)
+		go func() {
+			for sig := range hupch {
+				super.logger.WithField("signal", sig).Info("caught signal")
+				if super.err == nil {
+					super.err = errNeedConfigReload
+				}
 				super.cancel()
 			}
 		}()
 
+		if cfgPath != "" && cfgPath != "-" && cfg.AutoReloadConfig {
+			go watchConfig(super.ctx, super.logger, cfgPath, copyConfig(cfg), func() {
+				if super.err == nil {
+					super.err = errNeedConfigReload
+				}
+				super.cancel()
+			})
+		}
+
 		err := super.run(cfg)
 		if err != nil {
 			super.logger.WithError(err).Warn("supervisor shut down")
+			if super.err == nil {
+				super.err = err
+			}
 		}
-		close(super.done)
 	}()
 }
 
+func (super *Supervisor) Wait() error {
+	<-super.done
+	return super.err
+}
+
 func (super *Supervisor) run(cfg *arvados.Config) error {
+	defer super.cancel()
+
 	cwd, err := os.Getwd()
 	if err != nil {
 		return err
@@ -528,7 +568,7 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 		if p == "0" {
 			p = nextPort(h)
 		}
-		cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p)}
+		cluster.Services.Controller.ExternalURL = arvados.URL{Scheme: "https", Host: net.JoinHostPort(h, p), Path: "/"}
 	}
 	for _, svc := range []*arvados.Service{
 		&cluster.Services.Controller,
@@ -549,18 +589,19 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 		if svc.ExternalURL.Host == "" {
 			if svc == &cluster.Services.Controller ||
 				svc == &cluster.Services.GitHTTP ||
+				svc == &cluster.Services.Health ||
 				svc == &cluster.Services.Keepproxy ||
 				svc == &cluster.Services.WebDAV ||
 				svc == &cluster.Services.WebDAVDownload ||
 				svc == &cluster.Services.Workbench1 {
-				svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+				svc.ExternalURL = arvados.URL{Scheme: "https", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}
 			} else if svc == &cluster.Services.Websocket {
-				svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}
+				svc.ExternalURL = arvados.URL{Scheme: "wss", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/websocket"}
 			}
 		}
 		if len(svc.InternalURLs) == 0 {
 			svc.InternalURLs = map[arvados.URL]arvados.ServiceInstance{
-				arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}: arvados.ServiceInstance{},
+				arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}: arvados.ServiceInstance{},
 			}
 		}
 	}
@@ -588,7 +629,7 @@ func (super *Supervisor) autofillConfig(cfg *arvados.Config) error {
 	}
 	if super.ClusterType == "test" {
 		// Add a second keepstore process.
-		cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost))}] = arvados.ServiceInstance{}
+		cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%s", super.ListenHost, nextPort(super.ListenHost)), Path: "/"}] = arvados.ServiceInstance{}
 
 		// Create a directory-backed volume for each keepstore
 		// process.
@@ -706,3 +747,67 @@ func waitForConnect(ctx context.Context, addr string) error {
 	}
 	return ctx.Err()
 }
+
+func copyConfig(cfg *arvados.Config) *arvados.Config {
+	pr, pw := io.Pipe()
+	go func() {
+		err := json.NewEncoder(pw).Encode(cfg)
+		if err != nil {
+			panic(err)
+		}
+		pw.Close()
+	}()
+	cfg2 := new(arvados.Config)
+	err := json.NewDecoder(pr).Decode(cfg2)
+	if err != nil {
+		panic(err)
+	}
+	return cfg2
+}
+
+func watchConfig(ctx context.Context, logger logrus.FieldLogger, cfgPath string, prevcfg *arvados.Config, fn func()) {
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		logger.WithError(err).Error("fsnotify setup failed")
+		return
+	}
+	defer watcher.Close()
+
+	err = watcher.Add(cfgPath)
+	if err != nil {
+		logger.WithError(err).Error("fsnotify watcher failed")
+		return
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case err, ok := <-watcher.Errors:
+			if !ok {
+				return
+			}
+			logger.WithError(err).Warn("fsnotify watcher reported error")
+		case _, ok := <-watcher.Events:
+			if !ok {
+				return
+			}
+			for len(watcher.Events) > 0 {
+				<-watcher.Events
+			}
+			loader := config.NewLoader(&bytes.Buffer{}, &logrus.Logger{Out: ioutil.Discard})
+			loader.Path = cfgPath
+			loader.SkipAPICalls = true
+			cfg, err := loader.Load()
+			if err != nil {
+				logger.WithError(err).Warn("error reloading config file after change detected; ignoring new config for now")
+			} else if reflect.DeepEqual(cfg, prevcfg) {
+				logger.Debug("config file changed but is still DeepEqual to the existing config")
+			} else {
+				logger.Debug("config changed, notifying supervisor")
+				fn()
+				prevcfg = cfg
+			}
+		}
+	}
+}
diff --git a/lib/config/cmd_test.go b/lib/config/cmd_test.go
index f4f2d5653b..74c3cc9694 100644
--- a/lib/config/cmd_test.go
+++ b/lib/config/cmd_test.go
@@ -148,7 +148,7 @@ Clusters:
 	code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
 	c.Check(code, check.Equals, 0)
 	c.Check(stdout.String(), check.Matches, `(?ms).*TimeoutBooting: 10m\n.*`)
-	c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345: {}\n.*`)
+	c.Check(stdout.String(), check.Matches, `(?ms).*http://localhost:12345/: {}\n.*`)
 }
 
 func (s *CommandSuite) TestDump_UnknownKey(c *check.C) {
@@ -162,7 +162,7 @@ Clusters:
 	code := DumpCommand.RunCommand("arvados config-dump", []string{"-config", "-"}, bytes.NewBufferString(in), &stdout, &stderr)
 	c.Check(code, check.Equals, 0)
 	c.Check(stderr.String(), check.Matches, `(?ms).*deprecated or unknown config entry: Clusters.z1234.UnknownKey.*`)
-	c.Check(stdout.String(), check.Matches, `(?ms)Clusters:\n  z1234:\n.*`)
+	c.Check(stdout.String(), check.Matches, `(?ms)(.*\n)?Clusters:\n  z1234:\n.*`)
 	c.Check(stdout.String(), check.Matches, `(?ms).*\n *ManagementToken: secret\n.*`)
 	c.Check(stdout.String(), check.Not(check.Matches), `(?ms).*UnknownKey.*`)
 }
diff --git a/lib/config/config.default.yml b/lib/config/config.default.yml
index fcccdd0634..907acdc878 100644
--- a/lib/config/config.default.yml
+++ b/lib/config/config.default.yml
@@ -440,6 +440,13 @@ Clusters:
       # or omitted, pages are processed serially.
       BalanceCollectionBuffers: 1000
 
+      # Maximum time for a rebalancing run. This ensures keep-balance
+      # eventually gives up and retries if, for example, a network
+      # error causes a hung connection that is never closed by the
+      # OS. It should be long enough that it doesn't interrupt a
+      # long-running balancing operation.
+      BalanceTimeout: 6h
+
       # Default lifetime for ephemeral collections: 2 weeks. This must not
       # be less than BlobSigningTTL.
       DefaultTrashLifetime: 336h
@@ -524,54 +531,163 @@ Clusters:
         MaxUUIDEntries:       1000
 
     Login:
-      # These settings are provided by your OAuth2 provider (eg
-      # Google) used to perform upstream authentication.
-      ProviderAppID: ""
-      ProviderAppSecret: ""
-
-      # (Experimental) Authenticate with Google, bypassing the
-      # SSO-provider gateway service. Use the Google Cloud console to
-      # enable the People API (APIs and Services > Enable APIs and
-      # services > Google People API > Enable), generate a Client ID
-      # and secret (APIs and Services > Credentials > Create
-      # credentials > OAuth client ID > Web application) and add your
-      # controller's /login URL (e.g.,
-      # "https://zzzzz.example.com/login") as an authorized redirect
-      # URL.
-      #
-      # Incompatible with ForceLegacyAPI14. ProviderAppID must be
-      # blank.
-      GoogleClientID: ""
-      GoogleClientSecret: ""
-
-      # Allow users to log in to existing accounts using any verified
-      # email address listed by their Google account. If true, the
-      # Google People API must be enabled in order for Google login to
-      # work. If false, only the primary email address will be used.
-      GoogleAlternateEmailAddresses: true
-
-      # (Experimental) Use PAM to authenticate logins, using the
-      # specified PAM service name.
-      #
-      # Cannot be used in combination with OAuth2 (ProviderAppID) or
-      # Google (GoogleClientID). Cannot be used on a cluster acting as
-      # a LoginCluster.
-      PAM: false
-      PAMService: arvados
-
-      # Domain name (e.g., "example.com") to use to construct the
-      # user's email address if PAM authentication returns a username
-      # with no "@". If empty, use the PAM username as the user's
-      # email address, whether or not it contains "@".
-      #
-      # Note that the email address is used as the primary key for
-      # user records when logging in. Therefore, if you change
-      # PAMDefaultEmailDomain after the initial installation, you
-      # should also update existing user records to reflect the new
-      # domain. Otherwise, next time those users log in, they will be
-      # given new accounts instead of accessing their existing
-      # accounts.
-      PAMDefaultEmailDomain: ""
+      # One of the following mechanisms (SSO, Google, PAM, LDAP, or
+      # LoginCluster) should be enabled; see
+      # https://doc.arvados.org/install/setup-login.html
+
+      Google:
+        # Authenticate with Google.
+        Enable: false
+
+        # Use the Google Cloud console to enable the People API (APIs
+        # and Services > Enable APIs and services > Google People API
+        # > Enable), generate a Client ID and secret (APIs and
+        # Services > Credentials > Create credentials > OAuth client
+        # ID > Web application) and add your controller's /login URL
+        # (e.g., "https://zzzzz.example.com/login") as an authorized
+        # redirect URL.
+        #
+        # Incompatible with ForceLegacyAPI14. ProviderAppID must be
+        # blank.
+        ClientID: ""
+        ClientSecret: ""
+
+        # Allow users to log in to existing accounts using any verified
+        # email address listed by their Google account. If true, the
+        # Google People API must be enabled in order for Google login to
+        # work. If false, only the primary email address will be used.
+        AlternateEmailAddresses: true
+
+      OpenIDConnect:
+        # Authenticate with an OpenID Connect provider.
+        Enable: false
+
+        # Issuer URL, e.g., "https://login.example.com".
+        #
+        # This must be exactly equal to the URL returned by the issuer
+        # itself in its config response ("isser" key). If the
+        # configured value is "https://example" and the provider
+        # returns "https://example:443" or "https://example/" then
+        # login will fail, even though those URLs are equivalent
+        # (RFC3986).
+        Issuer: ""
+
+        # Your client ID and client secret (supplied by the provider).
+        ClientID: ""
+        ClientSecret: ""
+
+        # OpenID claim field containing the user's email
+        # address. Normally "email"; see
+        # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+        EmailClaim: "email"
+
+        # OpenID claim field containing the email verification
+        # flag. Normally "email_verified".  To accept every returned
+        # email address without checking a "verified" field at all,
+        # use the empty string "".
+        EmailVerifiedClaim: "email_verified"
+
+        # OpenID claim field containing the user's preferred
+        # username. If empty, use the mailbox part of the user's email
+        # address.
+        UsernameClaim: ""
+
+      PAM:
+        # (Experimental) Use PAM to authenticate users.
+        Enable: false
+
+        # PAM service name. PAM will apply the policy in the
+        # corresponding config file (e.g., /etc/pam.d/arvados) or, if
+        # there is none, the default "other" config.
+        Service: arvados
+
+        # Domain name (e.g., "example.com") to use to construct the
+        # user's email address if PAM authentication returns a
+        # username with no "@". If empty, use the PAM username as the
+        # user's email address, whether or not it contains "@".
+        #
+        # Note that the email address is used as the primary key for
+        # user records when logging in. Therefore, if you change
+        # PAMDefaultEmailDomain after the initial installation, you
+        # should also update existing user records to reflect the new
+        # domain. Otherwise, next time those users log in, they will
+        # be given new accounts instead of accessing their existing
+        # accounts.
+        DefaultEmailDomain: ""
+
+      LDAP:
+        # Use an LDAP service to authenticate users.
+        Enable: false
+
+        # Server URL, like "ldap://ldapserver.example.com:389" or
+        # "ldaps://ldapserver.example.com:636".
+        URL: "ldap://ldap:389"
+
+        # Use StartTLS upon connecting to the server.
+        StartTLS: true
+
+        # Skip TLS certificate name verification.
+        InsecureTLS: false
+
+        # Strip the @domain part if a user supplies an email-style
+        # username with this domain. If "*", strip any user-provided
+        # domain. If "", never strip the domain part. Example:
+        # "example.com"
+        StripDomain: ""
+
+        # If, after applying StripDomain, the username contains no "@"
+        # character, append this domain to form an email-style
+        # username. Example: "example.com"
+        AppendDomain: ""
+
+        # The LDAP attribute to filter on when looking up a username
+        # (after applying StripDomain and AppendDomain).
+        SearchAttribute: uid
+
+        # Bind with this username (DN or UPN) and password when
+        # looking up the user record.
+        #
+        # Example user: "cn=admin,dc=example,dc=com"
+        SearchBindUser: ""
+        SearchBindPassword: ""
+
+        # Directory base for username lookup. Example:
+        # "ou=Users,dc=example,dc=com"
+        SearchBase: ""
+
+        # Additional filters to apply when looking up users' LDAP
+        # entries. This can be used to restrict access to a subset of
+        # LDAP users, or to disambiguate users from other directory
+        # entries that have the SearchAttribute present.
+        #
+        # Special characters in assertion values must be escaped (see
+        # RFC4515).
+        #
+        # Example: "(objectClass=person)"
+        SearchFilters: ""
+
+        # LDAP attribute to use as the user's email address.
+        #
+        # Important: This must not be an attribute whose value can be
+        # edited in the directory by the users themselves. Otherwise,
+        # users can take over other users' Arvados accounts trivially
+        # (email address is the primary key for Arvados accounts.)
+        EmailAttribute: mail
+
+        # LDAP attribute to use as the preferred Arvados username. If
+        # no value is found (or this config is empty) the username
+        # originally supplied by the user will be used.
+        UsernameAttribute: uid
+
+      SSO:
+        # Authenticate with a separate SSO server. (Deprecated)
+        Enable: false
+
+        # ProviderAppID and ProviderAppSecret are generated during SSO
+        # setup; see
+        # https://doc.arvados.org/v2.0/install/install-sso.html#update-config
+        ProviderAppID: ""
+        ProviderAppSecret: ""
 
       # The cluster ID to delegate the user database.  When set,
       # logins on this cluster will be redirected to the login cluster
@@ -951,6 +1067,7 @@ Clusters:
           Region: us-east-1a
           Bucket: aaaaa
           LocationConstraint: false
+          V2Signature: false
           IndexPageSize: 1000
           ConnectTimeout: 1m
           ReadTimeout: 10m
@@ -1140,7 +1257,7 @@ Clusters:
       RunningJobLogRecordsToFetch: 2000
 
       # In systems with many shared projects, loading of dashboard and topnav
-      # cab be slow due to collections indexing; use the following parameters
+      # can be slow due to collections indexing; use the following parameters
       # to suppress these properties
       ShowRecentCollectionsOnDashboard: true
       ShowUserNotifications: true
@@ -1228,3 +1345,8 @@ Clusters:
     # implementation. Note that it also disables some new federation
     # features and will be removed in a future release.
     ForceLegacyAPI14: false
+
+# (Experimental) Restart services automatically when config file
+# changes are detected. Only supported by `arvados-server boot` in
+# dev/test mode.
+AutoReloadConfig: false
diff --git a/lib/config/deprecated.go b/lib/config/deprecated.go
index 0689efa440..1be7208ee3 100644
--- a/lib/config/deprecated.go
+++ b/lib/config/deprecated.go
@@ -23,6 +23,13 @@ type deprRequestLimits struct {
 type deprCluster struct {
 	RequestLimits deprRequestLimits
 	NodeProfiles  map[string]nodeProfile
+	Login         struct {
+		GoogleClientID                *string
+		GoogleClientSecret            *string
+		GoogleAlternateEmailAddresses *bool
+		ProviderAppID                 *string
+		ProviderAppSecret             *string
+	}
 }
 
 type deprecatedConfig struct {
@@ -80,6 +87,34 @@ func (ldr *Loader) applyDeprecatedConfig(cfg *arvados.Config) error {
 		if dst, n := &cluster.API.MaxRequestAmplification, dcluster.RequestLimits.MultiClusterRequestConcurrency; n != nil && *n != *dst {
 			*dst = *n
 		}
+
+		// Google* moved to Google.*
+		if dst, n := &cluster.Login.Google.ClientID, dcluster.Login.GoogleClientID; n != nil && *n != *dst {
+			*dst = *n
+			if *n != "" {
+				// In old config, non-empty ClientID meant enable
+				cluster.Login.Google.Enable = true
+			}
+		}
+		if dst, n := &cluster.Login.Google.ClientSecret, dcluster.Login.GoogleClientSecret; n != nil && *n != *dst {
+			*dst = *n
+		}
+		if dst, n := &cluster.Login.Google.AlternateEmailAddresses, dcluster.Login.GoogleAlternateEmailAddresses; n != nil && *n != *dst {
+			*dst = *n
+		}
+
+		// Provider* moved to SSO.Provider*
+		if dst, n := &cluster.Login.SSO.ProviderAppID, dcluster.Login.ProviderAppID; n != nil && *n != *dst {
+			*dst = *n
+			if *n != "" {
+				// In old config, non-empty ID meant enable
+				cluster.Login.SSO.Enable = true
+			}
+		}
+		if dst, n := &cluster.Login.SSO.ProviderAppSecret, dcluster.Login.ProviderAppSecret; n != nil && *n != *dst {
+			*dst = *n
+		}
+
 		cfg.Clusters[id] = cluster
 	}
 	return nil
@@ -100,7 +135,7 @@ func applyDeprecatedNodeProfile(hostname string, ssi systemServiceInstance, svc
 	if strings.HasPrefix(host, ":") {
 		host = hostname + host
 	}
-	svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host}] = arvados.ServiceInstance{}
+	svc.InternalURLs[arvados.URL{Scheme: scheme, Host: host, Path: "/"}] = arvados.ServiceInstance{}
 }
 
 func (ldr *Loader) loadOldConfigHelper(component, path string, target interface{}) error {
@@ -153,6 +188,7 @@ func loadOldClientConfig(cluster *arvados.Cluster, client *arvados.Client) {
 	}
 	if client.APIHost != "" {
 		cluster.Services.Controller.ExternalURL.Host = client.APIHost
+		cluster.Services.Controller.ExternalURL.Path = "/"
 	}
 	if client.Scheme != "" {
 		cluster.Services.Controller.ExternalURL.Scheme = client.Scheme
@@ -268,7 +304,7 @@ func (ldr *Loader) loadOldWebsocketConfig(cfg *arvados.Config) error {
 		cluster.PostgreSQL.ConnectionPool = *oc.PostgresPool
 	}
 	if oc.Listen != nil {
-		cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+		cluster.Services.Websocket.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
 	}
 	if oc.LogLevel != nil {
 		cluster.SystemLogs.LogLevel = *oc.LogLevel
@@ -327,7 +363,7 @@ func (ldr *Loader) loadOldKeepproxyConfig(cfg *arvados.Config) error {
 	loadOldClientConfig(cluster, oc.Client)
 
 	if oc.Listen != nil {
-		cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+		cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
 	}
 	if oc.DefaultReplicas != nil {
 		cluster.Collections.DefaultReplication = *oc.DefaultReplicas
@@ -413,11 +449,11 @@ func (ldr *Loader) loadOldKeepWebConfig(cfg *arvados.Config) error {
 	loadOldClientConfig(cluster, oc.Client)
 
 	if oc.Listen != nil {
-		cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
-		cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen}] = arvados.ServiceInstance{}
+		cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
+		cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: *oc.Listen, Path: "/"}] = arvados.ServiceInstance{}
 	}
 	if oc.AttachmentOnlyHost != nil {
-		cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost}
+		cluster.Services.WebDAVDownload.ExternalURL = arvados.URL{Host: *oc.AttachmentOnlyHost, Path: "/"}
 	}
 	if oc.ManagementToken != nil {
 		cluster.ManagementToken = *oc.ManagementToken
diff --git a/lib/config/deprecated_keepstore.go b/lib/config/deprecated_keepstore.go
index 401764c87a..186ffc3371 100644
--- a/lib/config/deprecated_keepstore.go
+++ b/lib/config/deprecated_keepstore.go
@@ -118,7 +118,7 @@ func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 		return err
 	}
 
-	myURL := arvados.URL{Scheme: "http"}
+	myURL := arvados.URL{Scheme: "http", Path: "/"}
 	if oc.TLSCertificateFile != nil && oc.TLSKeyFile != nil {
 		myURL.Scheme = "https"
 	}
@@ -137,7 +137,7 @@ func (ldr *Loader) loadOldKeepstoreConfig(cfg *arvados.Config) error {
 		cluster.TLS.Key = "file://" + *v
 	}
 	if v := oc.Listen; v != nil {
-		if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v}]; ok {
+		if _, ok := cluster.Services.Keepstore.InternalURLs[arvados.URL{Scheme: myURL.Scheme, Host: *v, Path: "/"}]; ok {
 			// already listed
 			myURL.Host = *v
 		} else if len(*v) > 1 && (*v)[0] == ':' {
@@ -537,6 +537,7 @@ func keepServiceURL(ks arvados.KeepService) arvados.URL {
 	url := arvados.URL{
 		Scheme: "http",
 		Host:   net.JoinHostPort(ks.ServiceHost, strconv.Itoa(ks.ServicePort)),
+		Path:   "/",
 	}
 	if ks.ServiceSSLFlag {
 		url.Scheme = "https"
diff --git a/lib/config/deprecated_keepstore_test.go b/lib/config/deprecated_keepstore_test.go
index abc507fdad..dab308c9d1 100644
--- a/lib/config/deprecated_keepstore_test.go
+++ b/lib/config/deprecated_keepstore_test.go
@@ -109,7 +109,7 @@ Clusters:
     TLS: {Insecure: true}
     Services:
       Controller:
-        ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
+        ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
 `, `
 Clusters:
   z1111:
@@ -120,7 +120,7 @@ Clusters:
         InternalURLs:
           "http://`+hostname+`:25107": {Rendezvous: `+s.ksByPort[25107].UUID[12:]+`}
       Controller:
-        ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`"
+        ExternalURL: "https://`+os.Getenv("ARVADOS_API_HOST")+`/"
     SystemLogs:
       Format: text
       LogLevel: debug
@@ -254,7 +254,7 @@ Volumes:
   ReadOnly: true
   StorageAccountName: storageacctname
   StorageAccountKeyFile: `+secretkeyfile.Name()+`
-  StorageBaseURL: https://example.example
+  StorageBaseURL: https://example.example/
   ContainerName: testctr
   LocationConstraint: true
   AzureReplication: 4
@@ -268,7 +268,7 @@ Volumes:
 	}, &arvados.AzureVolumeDriverParameters{
 		StorageAccountName:   "storageacctname",
 		StorageAccountKey:    "secretkeydata",
-		StorageBaseURL:       "https://example.example",
+		StorageBaseURL:       "https://example.example/",
 		ContainerName:        "testctr",
 		RequestTimeout:       arvados.Duration(time.Minute * 3),
 		ListBlobsRetryDelay:  arvados.Duration(time.Minute * 4),
@@ -333,7 +333,7 @@ func (s *KeepstoreMigrationSuite) testDeprecatedVolume(c *check.C, oldconfigdata
 		c.Check(v.Driver, check.Equals, expectvol.Driver)
 		c.Check(v.Replication, check.Equals, expectvol.Replication)
 
-		avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345"}]
+		avh, ok := v.AccessViaHosts[arvados.URL{Scheme: "http", Host: hostname + ":12345", Path: "/"}]
 		c.Check(ok, check.Equals, true)
 		c.Check(avh.ReadOnly, check.Equals, expectvol.ReadOnly)
 
@@ -516,6 +516,7 @@ Volumes:
 	url := arvados.URL{
 		Scheme: "http",
 		Host:   fmt.Sprintf("%s:%d", hostname, port),
+		Path:   "/",
 	}
 	_, ok := before["zzzzz-nyw5e-readonlyonother"].AccessViaHosts[url]
 	c.Check(ok, check.Equals, false)
@@ -543,6 +544,7 @@ Volumes:
 	url := arvados.URL{
 		Scheme: "http",
 		Host:   fmt.Sprintf("%s:%d", hostname, port),
+		Path:   "/",
 	}
 	_, ok := before["zzzzz-nyw5e-writableonother"].AccessViaHosts[url]
 	c.Check(ok, check.Equals, false)
@@ -572,7 +574,7 @@ Volumes:
 
 	hostname, err := os.Hostname()
 	c.Assert(err, check.IsNil)
-	_, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port)}]
+	_, ok := newvol.AccessViaHosts[arvados.URL{Scheme: "http", Host: fmt.Sprintf("%s:%d", hostname, port), Path: "/"}]
 	c.Check(ok, check.Equals, true)
 }
 
@@ -601,7 +603,7 @@ Volumes:
 	c.Check(logs, check.Matches, `(?ms).*you should remove the legacy keepstore config file.*`)
 	c.Check(logs, check.Matches, `(?ms).*you should migrate the legacy keepstore configuration file on host keep1.zzzzz.example.com.*`)
 	c.Check(logs, check.Not(check.Matches), `(?ms).*should migrate.*keep0.zzzzz.example.com.*`)
-	c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107 does not have access to any volumes.*`)
+	c.Check(logs, check.Matches, `(?ms).*keepstore configured at http://keep2.zzzzz.example.com:25107/ does not have access to any volumes.*`)
 	c.Check(logs, check.Matches, `(?ms).*Volumes.zzzzz-nyw5e-possconfigerror.AccessViaHosts refers to nonexistent keepstore server http://keep00.zzzzz.example.com:25107.*`)
 }
 
diff --git a/lib/config/deprecated_test.go b/lib/config/deprecated_test.go
index 58c27e984a..87e26fd096 100644
--- a/lib/config/deprecated_test.go
+++ b/lib/config/deprecated_test.go
@@ -89,6 +89,41 @@ Clusters:
 `)
 }
 
+func (s *LoadSuite) TestDeprecatedLoginBackend(c *check.C) {
+	checkEquivalent(c, `
+Clusters:
+ z1111:
+  Login:
+   GoogleClientID: aaaa
+   GoogleClientSecret: bbbb
+   GoogleAlternateEmailAddresses: true
+`, `
+Clusters:
+ z1111:
+  Login:
+   Google:
+    Enable: true
+    ClientID: aaaa
+    ClientSecret: bbbb
+    AlternateEmailAddresses: true
+`)
+	checkEquivalent(c, `
+Clusters:
+ z1111:
+  Login:
+   ProviderAppID: aaaa
+   ProviderAppSecret: bbbb
+`, `
+Clusters:
+ z1111:
+  Login:
+   SSO:
+    Enable: true
+    ProviderAppID: aaaa
+    ProviderAppSecret: bbbb
+`)
+}
+
 func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 	content := []byte(`
 {
@@ -117,7 +152,7 @@ func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 	cluster, err := testLoadLegacyConfig(content, "-legacy-keepweb-config", c)
 	c.Check(err, check.IsNil)
 
-	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 	c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 
 	c.Check(cluster.Collections.WebDAVCache.TTL, check.Equals, arvados.Duration(60*time.Second))
@@ -127,7 +162,7 @@ func (s *LoadSuite) TestLegacyKeepWebConfig(c *check.C) {
 	c.Check(cluster.Collections.WebDAVCache.MaxPermissionEntries, check.Equals, 100)
 	c.Check(cluster.Collections.WebDAVCache.MaxUUIDEntries, check.Equals, 100)
 
-	c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com"})
+	c.Check(cluster.Services.WebDAVDownload.ExternalURL, check.Equals, arvados.URL{Host: "download.example.com", Path: "/"})
 	c.Check(cluster.Services.WebDAVDownload.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 	c.Check(cluster.Services.WebDAV.InternalURLs[arvados.URL{Host: ":80"}], check.NotNil)
 
@@ -160,7 +195,7 @@ func (s *LoadSuite) TestLegacyKeepproxyConfig(c *check.C) {
 
 	c.Check(err, check.IsNil)
 	c.Check(cluster, check.NotNil)
-	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 	c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 	c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 	c.Check(cluster.Services.Keepproxy.InternalURLs[arvados.URL{Host: ":80"}], check.Equals, arvados.ServiceInstance{})
@@ -228,7 +263,7 @@ func (s *LoadSuite) TestLegacyArvGitHttpdConfig(c *check.C) {
 
 	c.Check(err, check.IsNil)
 	c.Check(cluster, check.NotNil)
-	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 	c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 	c.Check(cluster.ManagementToken, check.Equals, "xyzzy")
 	c.Check(cluster.Git.GitCommand, check.Equals, "/test/git")
diff --git a/lib/config/export.go b/lib/config/export.go
index ded03fc303..d6b02b750d 100644
--- a/lib/config/export.go
+++ b/lib/config/export.go
@@ -102,6 +102,7 @@ var whitelist = map[string]bool{
 	"Collections.WebDAVCache":                      false,
 	"Collections.BalanceCollectionBatch":           false,
 	"Collections.BalancePeriod":                    false,
+	"Collections.BalanceTimeout":                   false,
 	"Collections.BlobMissingReport":                false,
 	"Collections.BalanceCollectionBuffers":         false,
 	"Containers":                                   true,
@@ -131,15 +132,42 @@ var whitelist = map[string]bool{
 	"InstanceTypes.*":                              true,
 	"InstanceTypes.*.*":                            true,
 	"Login":                                        true,
-	"Login.GoogleClientID":                         false,
-	"Login.GoogleClientSecret":                     false,
-	"Login.GoogleAlternateEmailAddresses":          false,
-	"Login.PAM":                                    true,
-	"Login.PAMService":                             false,
-	"Login.PAMDefaultEmailDomain":                  false,
-	"Login.ProviderAppID":                          false,
-	"Login.ProviderAppSecret":                      false,
+	"Login.Google":                                 true,
+	"Login.Google.AlternateEmailAddresses":         false,
+	"Login.Google.ClientID":                        false,
+	"Login.Google.ClientSecret":                    false,
+	"Login.Google.Enable":                          true,
+	"Login.LDAP":                                   true,
+	"Login.LDAP.AppendDomain":                      false,
+	"Login.LDAP.EmailAttribute":                    false,
+	"Login.LDAP.Enable":                            true,
+	"Login.LDAP.InsecureTLS":                       false,
+	"Login.LDAP.SearchAttribute":                   false,
+	"Login.LDAP.SearchBase":                        false,
+	"Login.LDAP.SearchBindPassword":                false,
+	"Login.LDAP.SearchBindUser":                    false,
+	"Login.LDAP.SearchFilters":                     false,
+	"Login.LDAP.StartTLS":                          false,
+	"Login.LDAP.StripDomain":                       false,
+	"Login.LDAP.URL":                               false,
+	"Login.LDAP.UsernameAttribute":                 false,
 	"Login.LoginCluster":                           true,
+	"Login.OpenIDConnect":                          true,
+	"Login.OpenIDConnect.ClientID":                 false,
+	"Login.OpenIDConnect.ClientSecret":             false,
+	"Login.OpenIDConnect.Enable":                   true,
+	"Login.OpenIDConnect.Issuer":                   false,
+	"Login.OpenIDConnect.EmailClaim":               false,
+	"Login.OpenIDConnect.EmailVerifiedClaim":       false,
+	"Login.OpenIDConnect.UsernameClaim":            false,
+	"Login.PAM":                                    true,
+	"Login.PAM.DefaultEmailDomain":                 false,
+	"Login.PAM.Enable":                             true,
+	"Login.PAM.Service":                            false,
+	"Login.SSO":                                    true,
+	"Login.SSO.Enable":                             true,
+	"Login.SSO.ProviderAppID":                      false,
+	"Login.SSO.ProviderAppSecret":                  false,
 	"Login.RemoteTokenRefresh":                     true,
 	"Mail":                                         true,
 	"Mail.MailchimpAPIKey":                         false,
diff --git a/lib/config/generated_config.go b/lib/config/generated_config.go
index 4a8d7024fb..96da19dfcd 100644
--- a/lib/config/generated_config.go
+++ b/lib/config/generated_config.go
@@ -446,6 +446,13 @@ Clusters:
       # or omitted, pages are processed serially.
       BalanceCollectionBuffers: 1000
 
+      # Maximum time for a rebalancing run. This ensures keep-balance
+      # eventually gives up and retries if, for example, a network
+      # error causes a hung connection that is never closed by the
+      # OS. It should be long enough that it doesn't interrupt a
+      # long-running balancing operation.
+      BalanceTimeout: 6h
+
       # Default lifetime for ephemeral collections: 2 weeks. This must not
       # be less than BlobSigningTTL.
       DefaultTrashLifetime: 336h
@@ -530,54 +537,163 @@ Clusters:
         MaxUUIDEntries:       1000
 
     Login:
-      # These settings are provided by your OAuth2 provider (eg
-      # Google) used to perform upstream authentication.
-      ProviderAppID: ""
-      ProviderAppSecret: ""
-
-      # (Experimental) Authenticate with Google, bypassing the
-      # SSO-provider gateway service. Use the Google Cloud console to
-      # enable the People API (APIs and Services > Enable APIs and
-      # services > Google People API > Enable), generate a Client ID
-      # and secret (APIs and Services > Credentials > Create
-      # credentials > OAuth client ID > Web application) and add your
-      # controller's /login URL (e.g.,
-      # "https://zzzzz.example.com/login") as an authorized redirect
-      # URL.
-      #
-      # Incompatible with ForceLegacyAPI14. ProviderAppID must be
-      # blank.
-      GoogleClientID: ""
-      GoogleClientSecret: ""
-
-      # Allow users to log in to existing accounts using any verified
-      # email address listed by their Google account. If true, the
-      # Google People API must be enabled in order for Google login to
-      # work. If false, only the primary email address will be used.
-      GoogleAlternateEmailAddresses: true
-
-      # (Experimental) Use PAM to authenticate logins, using the
-      # specified PAM service name.
-      #
-      # Cannot be used in combination with OAuth2 (ProviderAppID) or
-      # Google (GoogleClientID). Cannot be used on a cluster acting as
-      # a LoginCluster.
-      PAM: false
-      PAMService: arvados
-
-      # Domain name (e.g., "example.com") to use to construct the
-      # user's email address if PAM authentication returns a username
-      # with no "@". If empty, use the PAM username as the user's
-      # email address, whether or not it contains "@".
-      #
-      # Note that the email address is used as the primary key for
-      # user records when logging in. Therefore, if you change
-      # PAMDefaultEmailDomain after the initial installation, you
-      # should also update existing user records to reflect the new
-      # domain. Otherwise, next time those users log in, they will be
-      # given new accounts instead of accessing their existing
-      # accounts.
-      PAMDefaultEmailDomain: ""
+      # One of the following mechanisms (SSO, Google, PAM, LDAP, or
+      # LoginCluster) should be enabled; see
+      # https://doc.arvados.org/install/setup-login.html
+
+      Google:
+        # Authenticate with Google.
+        Enable: false
+
+        # Use the Google Cloud console to enable the People API (APIs
+        # and Services > Enable APIs and services > Google People API
+        # > Enable), generate a Client ID and secret (APIs and
+        # Services > Credentials > Create credentials > OAuth client
+        # ID > Web application) and add your controller's /login URL
+        # (e.g., "https://zzzzz.example.com/login") as an authorized
+        # redirect URL.
+        #
+        # Incompatible with ForceLegacyAPI14. ProviderAppID must be
+        # blank.
+        ClientID: ""
+        ClientSecret: ""
+
+        # Allow users to log in to existing accounts using any verified
+        # email address listed by their Google account. If true, the
+        # Google People API must be enabled in order for Google login to
+        # work. If false, only the primary email address will be used.
+        AlternateEmailAddresses: true
+
+      OpenIDConnect:
+        # Authenticate with an OpenID Connect provider.
+        Enable: false
+
+        # Issuer URL, e.g., "https://login.example.com".
+        #
+        # This must be exactly equal to the URL returned by the issuer
+        # itself in its config response ("isser" key). If the
+        # configured value is "https://example" and the provider
+        # returns "https://example:443" or "https://example/" then
+        # login will fail, even though those URLs are equivalent
+        # (RFC3986).
+        Issuer: ""
+
+        # Your client ID and client secret (supplied by the provider).
+        ClientID: ""
+        ClientSecret: ""
+
+        # OpenID claim field containing the user's email
+        # address. Normally "email"; see
+        # https://openid.net/specs/openid-connect-core-1_0.html#StandardClaims
+        EmailClaim: "email"
+
+        # OpenID claim field containing the email verification
+        # flag. Normally "email_verified".  To accept every returned
+        # email address without checking a "verified" field at all,
+        # use the empty string "".
+        EmailVerifiedClaim: "email_verified"
+
+        # OpenID claim field containing the user's preferred
+        # username. If empty, use the mailbox part of the user's email
+        # address.
+        UsernameClaim: ""
+
+      PAM:
+        # (Experimental) Use PAM to authenticate users.
+        Enable: false
+
+        # PAM service name. PAM will apply the policy in the
+        # corresponding config file (e.g., /etc/pam.d/arvados) or, if
+        # there is none, the default "other" config.
+        Service: arvados
+
+        # Domain name (e.g., "example.com") to use to construct the
+        # user's email address if PAM authentication returns a
+        # username with no "@". If empty, use the PAM username as the
+        # user's email address, whether or not it contains "@".
+        #
+        # Note that the email address is used as the primary key for
+        # user records when logging in. Therefore, if you change
+        # PAMDefaultEmailDomain after the initial installation, you
+        # should also update existing user records to reflect the new
+        # domain. Otherwise, next time those users log in, they will
+        # be given new accounts instead of accessing their existing
+        # accounts.
+        DefaultEmailDomain: ""
+
+      LDAP:
+        # Use an LDAP service to authenticate users.
+        Enable: false
+
+        # Server URL, like "ldap://ldapserver.example.com:389" or
+        # "ldaps://ldapserver.example.com:636".
+        URL: "ldap://ldap:389"
+
+        # Use StartTLS upon connecting to the server.
+        StartTLS: true
+
+        # Skip TLS certificate name verification.
+        InsecureTLS: false
+
+        # Strip the @domain part if a user supplies an email-style
+        # username with this domain. If "*", strip any user-provided
+        # domain. If "", never strip the domain part. Example:
+        # "example.com"
+        StripDomain: ""
+
+        # If, after applying StripDomain, the username contains no "@"
+        # character, append this domain to form an email-style
+        # username. Example: "example.com"
+        AppendDomain: ""
+
+        # The LDAP attribute to filter on when looking up a username
+        # (after applying StripDomain and AppendDomain).
+        SearchAttribute: uid
+
+        # Bind with this username (DN or UPN) and password when
+        # looking up the user record.
+        #
+        # Example user: "cn=admin,dc=example,dc=com"
+        SearchBindUser: ""
+        SearchBindPassword: ""
+
+        # Directory base for username lookup. Example:
+        # "ou=Users,dc=example,dc=com"
+        SearchBase: ""
+
+        # Additional filters to apply when looking up users' LDAP
+        # entries. This can be used to restrict access to a subset of
+        # LDAP users, or to disambiguate users from other directory
+        # entries that have the SearchAttribute present.
+        #
+        # Special characters in assertion values must be escaped (see
+        # RFC4515).
+        #
+        # Example: "(objectClass=person)"
+        SearchFilters: ""
+
+        # LDAP attribute to use as the user's email address.
+        #
+        # Important: This must not be an attribute whose value can be
+        # edited in the directory by the users themselves. Otherwise,
+        # users can take over other users' Arvados accounts trivially
+        # (email address is the primary key for Arvados accounts.)
+        EmailAttribute: mail
+
+        # LDAP attribute to use as the preferred Arvados username. If
+        # no value is found (or this config is empty) the username
+        # originally supplied by the user will be used.
+        UsernameAttribute: uid
+
+      SSO:
+        # Authenticate with a separate SSO server. (Deprecated)
+        Enable: false
+
+        # ProviderAppID and ProviderAppSecret are generated during SSO
+        # setup; see
+        # https://doc.arvados.org/v2.0/install/install-sso.html#update-config
+        ProviderAppID: ""
+        ProviderAppSecret: ""
 
       # The cluster ID to delegate the user database.  When set,
       # logins on this cluster will be redirected to the login cluster
@@ -957,6 +1073,7 @@ Clusters:
           Region: us-east-1a
           Bucket: aaaaa
           LocationConstraint: false
+          V2Signature: false
           IndexPageSize: 1000
           ConnectTimeout: 1m
           ReadTimeout: 10m
@@ -1146,7 +1263,7 @@ Clusters:
       RunningJobLogRecordsToFetch: 2000
 
       # In systems with many shared projects, loading of dashboard and topnav
-      # cab be slow due to collections indexing; use the following parameters
+      # can be slow due to collections indexing; use the following parameters
       # to suppress these properties
       ShowRecentCollectionsOnDashboard: true
       ShowUserNotifications: true
@@ -1234,4 +1351,9 @@ Clusters:
     # implementation. Note that it also disables some new federation
     # features and will be removed in a future release.
     ForceLegacyAPI14: false
+
+# (Experimental) Restart services automatically when config file
+# changes are detected. Only supported by ` + "`" + `arvados-server boot` + "`" + ` in
+# dev/test mode.
+AutoReloadConfig: false
 `)
diff --git a/lib/config/load.go b/lib/config/load.go
index 86a8f7df6d..be6181bbe9 100644
--- a/lib/config/load.go
+++ b/lib/config/load.go
@@ -64,14 +64,16 @@ func NewLoader(stdin io.Reader, logger logrus.FieldLogger) *Loader {
 //	// ldr.Path == "/tmp/c.yaml"
 func (ldr *Loader) SetupFlags(flagset *flag.FlagSet) {
 	flagset.StringVar(&ldr.Path, "config", arvados.DefaultConfigFile, "Site configuration `file` (default may be overridden by setting an ARVADOS_CONFIG environment variable)")
-	flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
-	flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
-	flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
-	flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
-	flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
-	flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
-	flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
-	flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
+	if !ldr.SkipLegacy {
+		flagset.StringVar(&ldr.KeepstorePath, "legacy-keepstore-config", defaultKeepstoreConfigPath, "Legacy keepstore configuration `file`")
+		flagset.StringVar(&ldr.KeepWebPath, "legacy-keepweb-config", defaultKeepWebConfigPath, "Legacy keep-web configuration `file`")
+		flagset.StringVar(&ldr.CrunchDispatchSlurmPath, "legacy-crunch-dispatch-slurm-config", defaultCrunchDispatchSlurmConfigPath, "Legacy crunch-dispatch-slurm configuration `file`")
+		flagset.StringVar(&ldr.WebsocketPath, "legacy-ws-config", defaultWebsocketConfigPath, "Legacy arvados-ws configuration `file`")
+		flagset.StringVar(&ldr.KeepproxyPath, "legacy-keepproxy-config", defaultKeepproxyConfigPath, "Legacy keepproxy configuration `file`")
+		flagset.StringVar(&ldr.GitHttpdPath, "legacy-git-httpd-config", defaultGitHttpdConfigPath, "Legacy arv-git-httpd configuration `file`")
+		flagset.StringVar(&ldr.KeepBalancePath, "legacy-keepbalance-config", defaultKeepBalanceConfigPath, "Legacy keep-balance configuration `file`")
+		flagset.BoolVar(&ldr.SkipLegacy, "skip-legacy", false, "Don't load legacy config files")
+	}
 }
 
 // MungeLegacyConfigArgs checks args for a -config flag whose argument
diff --git a/lib/controller/api/routable.go b/lib/controller/api/routable.go
new file mode 100644
index 0000000000..6049cba8e4
--- /dev/null
+++ b/lib/controller/api/routable.go
@@ -0,0 +1,17 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+// Package api provides types used by controller/server-component
+// packages.
+package api
+
+import "context"
+
+// A RoutableFunc calls an API method (sometimes via a wrapped
+// RoutableFunc) that has real argument types.
+//
+// (It is used by ctrlctx to manage database transactions, so moving
+// it to the router package would cause a circular dependency
+// router->arvadostest->ctrlctx->router.)
+type RoutableFunc func(ctx context.Context, opts interface{}) (interface{}, error)
diff --git a/lib/controller/fed_containers.go b/lib/controller/fed_containers.go
index a923f757f2..c62cea1168 100644
--- a/lib/controller/fed_containers.go
+++ b/lib/controller/fed_containers.go
@@ -42,13 +42,11 @@ func remoteContainerRequestCreate(
 		return true
 	}
 
-	if *clusterId == "" {
-		*clusterId = h.handler.Cluster.ClusterID
-	}
-
-	if strings.HasPrefix(currentUser.Authorization.UUID, h.handler.Cluster.ClusterID) &&
-		*clusterId == h.handler.Cluster.ClusterID {
-		// local user submitting container request to local cluster
+	if *clusterId == "" || *clusterId == h.handler.Cluster.ClusterID {
+		// Submitting container request to local cluster. No
+		// need to set a runtime_token (rails api will create
+		// one when the container runs) or do a remote cluster
+		// request.
 		return false
 	}
 
diff --git a/lib/controller/federation.go b/lib/controller/federation.go
index 674183dcc1..aceaba8087 100644
--- a/lib/controller/federation.go
+++ b/lib/controller/federation.go
@@ -19,6 +19,7 @@ import (
 
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"github.com/jmcvetta/randutil"
 )
 
@@ -151,8 +152,9 @@ type CurrentUser struct {
 // non-nil, true, nil -- if the token is valid
 func (h *Handler) validateAPItoken(req *http.Request, token string) (*CurrentUser, bool, error) {
 	user := CurrentUser{Authorization: arvados.APIClientAuthorization{APIToken: token}}
-	db, err := h.db(req)
+	db, err := h.db(req.Context())
 	if err != nil {
+		ctxlog.FromContext(req.Context()).WithError(err).Debugf("validateAPItoken(%s): database error", token)
 		return nil, false, err
 	}
 
@@ -164,25 +166,30 @@ func (h *Handler) validateAPItoken(req *http.Request, token string) (*CurrentUse
 	}
 	user.Authorization.APIToken = token
 	var scopes string
-	err = db.QueryRowContext(req.Context(), `SELECT api_client_authorizations.uuid, api_client_authorizations.scopes, users.uuid FROM api_client_authorizations JOIN users on api_client_authorizations.user_id=users.id WHERE api_token=$1 AND (expires_at IS NULL OR expires_at > current_timestamp) LIMIT 1`, token).Scan(&user.Authorization.UUID, &scopes, &user.UUID)
+	err = db.QueryRowContext(req.Context(), `SELECT api_client_authorizations.uuid, api_client_authorizations.scopes, users.uuid FROM api_client_authorizations JOIN users on api_client_authorizations.user_id=users.id WHERE api_token=$1 AND (expires_at IS NULL OR expires_at > current_timestamp AT TIME ZONE 'UTC') LIMIT 1`, token).Scan(&user.Authorization.UUID, &scopes, &user.UUID)
 	if err == sql.ErrNoRows {
+		ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): not found in database", token)
 		return nil, false, nil
 	} else if err != nil {
+		ctxlog.FromContext(req.Context()).WithError(err).Debugf("validateAPItoken(%s): database error", token)
 		return nil, false, err
 	}
 	if uuid != "" && user.Authorization.UUID != uuid {
 		// secret part matches, but UUID doesn't -- somewhat surprising
+		ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): secret part found, but with different UUID: %s", token, user.Authorization.UUID)
 		return nil, false, nil
 	}
 	err = json.Unmarshal([]byte(scopes), &user.Authorization.Scopes)
 	if err != nil {
+		ctxlog.FromContext(req.Context()).WithError(err).Debugf("validateAPItoken(%s): error parsing scopes from db", token)
 		return nil, false, err
 	}
+	ctxlog.FromContext(req.Context()).Debugf("validateAPItoken(%s): ok", token)
 	return &user, true, nil
 }
 
 func (h *Handler) createAPItoken(req *http.Request, userUUID string, scopes []string) (*arvados.APIClientAuthorization, error) {
-	db, err := h.db(req)
+	db, err := h.db(req.Context())
 	if err != nil {
 		return nil, err
 	}
@@ -207,9 +214,9 @@ func (h *Handler) createAPItoken(req *http.Request, userUUID string, scopes []st
 (uuid, api_token, expires_at, scopes,
 user_id,
 api_client_id, created_at, updated_at)
-VALUES ($1, $2, CURRENT_TIMESTAMP + INTERVAL '2 weeks', $3,
+VALUES ($1, $2, CURRENT_TIMESTAMP AT TIME ZONE 'UTC' + INTERVAL '2 weeks', $3,
 (SELECT id FROM users WHERE users.uuid=$4 LIMIT 1),
-0, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`,
+0, CURRENT_TIMESTAMP AT TIME ZONE 'UTC', CURRENT_TIMESTAMP AT TIME ZONE 'UTC')`,
 		uuid, token, string(scopesjson), userUUID)
 
 	if err != nil {
diff --git a/lib/controller/federation/federation_test.go b/lib/controller/federation/federation_test.go
index f57d827848..256afc8e6b 100644
--- a/lib/controller/federation/federation_test.go
+++ b/lib/controller/federation/federation_test.go
@@ -64,7 +64,7 @@ func (s *FederationSuite) addDirectRemote(c *check.C, id string, backend backend
 
 func (s *FederationSuite) addHTTPRemote(c *check.C, id string, backend backend) {
 	srv := httpserver.Server{Addr: ":"}
-	srv.Handler = router.New(backend)
+	srv.Handler = router.New(backend, nil)
 	c.Check(srv.Start(), check.IsNil)
 	s.cluster.RemoteClusters[id] = arvados.RemoteCluster{
 		Scheme: "http",
diff --git a/lib/controller/federation/login_test.go b/lib/controller/federation/login_test.go
index 2de260fdc2..ad91bcf802 100644
--- a/lib/controller/federation/login_test.go
+++ b/lib/controller/federation/login_test.go
@@ -43,7 +43,8 @@ func (s *LoginSuite) TestDeferToLoginCluster(c *check.C) {
 func (s *LoginSuite) TestLogout(c *check.C) {
 	s.cluster.Services.Workbench1.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench1.example.com"}
 	s.cluster.Services.Workbench2.ExternalURL = arvados.URL{Scheme: "https", Host: "workbench2.example.com"}
-	s.cluster.Login.GoogleClientID = "zzzzzzzzzzzzzz"
+	s.cluster.Login.Google.Enable = true
+	s.cluster.Login.Google.ClientID = "zzzzzzzzzzzzzz"
 	s.addHTTPRemote(c, "zhome", &arvadostest.APIStub{})
 	s.cluster.Login.LoginCluster = "zhome"
 	// s.fed is already set by SetUpTest, but we need to
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index 2b0cb22b04..6a9ad8c15f 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -64,6 +64,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
 	cluster.TLS.Insecure = true
 	cluster.API.MaxItemsPerResponse = 1000
 	cluster.API.MaxRequestAmplification = 4
+	cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
 	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
 	arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
 	s.testHandler = &Handler{Cluster: cluster}
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 01f2161632..e742bbc59b 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -6,7 +6,6 @@ package controller
 
 import (
 	"context"
-	"database/sql"
 	"errors"
 	"fmt"
 	"net/http"
@@ -18,9 +17,12 @@ import (
 	"git.arvados.org/arvados.git/lib/controller/federation"
 	"git.arvados.org/arvados.git/lib/controller/railsproxy"
 	"git.arvados.org/arvados.git/lib/controller/router"
+	"git.arvados.org/arvados.git/lib/ctrlctx"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"git.arvados.org/arvados.git/sdk/go/health"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
+	"github.com/jmoiron/sqlx"
 	_ "github.com/lib/pq"
 )
 
@@ -32,7 +34,7 @@ type Handler struct {
 	proxy          *proxy
 	secureClient   *http.Client
 	insecureClient *http.Client
-	pgdb           *sql.DB
+	pgdb           *sqlx.DB
 	pgdbMtx        sync.Mutex
 }
 
@@ -63,7 +65,11 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 
 func (h *Handler) CheckHealth() error {
 	h.setupOnce.Do(h.setup)
-	_, _, err := railsproxy.FindRailsAPI(h.Cluster)
+	_, err := h.db(context.TODO())
+	if err != nil {
+		return err
+	}
+	_, _, err = railsproxy.FindRailsAPI(h.Cluster)
 	return err
 }
 
@@ -78,10 +84,10 @@ func (h *Handler) setup() {
 	mux.Handle("/_health/", &health.Handler{
 		Token:  h.Cluster.ManagementToken,
 		Prefix: "/_health/",
-		Routes: health.Routes{"ping": func() error { _, err := h.db(&http.Request{}); return err }},
+		Routes: health.Routes{"ping": func() error { _, err := h.db(context.TODO()); return err }},
 	})
 
-	rtr := router.New(federation.New(h.Cluster))
+	rtr := router.New(federation.New(h.Cluster), ctrlctx.WrapCallsInTransactions(h.db))
 	mux.Handle("/arvados/v1/config", rtr)
 	mux.Handle("/"+arvados.EndpointUserAuthenticate.Path, rtr)
 
@@ -115,23 +121,23 @@ func (h *Handler) setup() {
 
 var errDBConnection = errors.New("database connection error")
 
-func (h *Handler) db(req *http.Request) (*sql.DB, error) {
+func (h *Handler) db(ctx context.Context) (*sqlx.DB, error) {
 	h.pgdbMtx.Lock()
 	defer h.pgdbMtx.Unlock()
 	if h.pgdb != nil {
 		return h.pgdb, nil
 	}
 
-	db, err := sql.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
+	db, err := sqlx.Open("postgres", h.Cluster.PostgreSQL.Connection.String())
 	if err != nil {
-		httpserver.Logger(req).WithError(err).Error("postgresql connect failed")
+		ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect failed")
 		return nil, errDBConnection
 	}
 	if p := h.Cluster.PostgreSQL.ConnectionPool; p > 0 {
 		db.SetMaxOpenConns(p)
 	}
 	if err := db.Ping(); err != nil {
-		httpserver.Logger(req).WithError(err).Error("postgresql connect succeeded but ping failed")
+		ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect scuceeded but ping failed")
 		return nil, errDBConnection
 	}
 	h.pgdb = db
diff --git a/lib/controller/handler_test.go b/lib/controller/handler_test.go
index f09203f724..ef6b9195f1 100644
--- a/lib/controller/handler_test.go
+++ b/lib/controller/handler_test.go
@@ -52,6 +52,7 @@ func (s *HandlerSuite) SetUpTest(c *check.C) {
 		PostgreSQL:       integrationTestCluster().PostgreSQL,
 		ForceLegacyAPI14: forceLegacyAPI14,
 	}
+	s.cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
 	s.cluster.TLS.Insecure = true
 	arvadostest.SetServiceURL(&s.cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 	arvadostest.SetServiceURL(&s.cluster.Services.Controller, "http://localhost:/")
@@ -71,7 +72,10 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) {
 		req := httptest.NewRequest(method, "/arvados/v1/config", nil)
 		resp := httptest.NewRecorder()
 		s.handler.ServeHTTP(resp, req)
-		c.Check(resp.Code, check.Equals, http.StatusOK)
+		c.Log(resp.Body.String())
+		if !c.Check(resp.Code, check.Equals, http.StatusOK) {
+			continue
+		}
 		c.Check(resp.Header().Get("Access-Control-Allow-Origin"), check.Equals, `*`)
 		c.Check(resp.Header().Get("Access-Control-Allow-Methods"), check.Matches, `.*\bGET\b.*`)
 		c.Check(resp.Header().Get("Access-Control-Allow-Headers"), check.Matches, `.+`)
@@ -80,12 +84,11 @@ func (s *HandlerSuite) TestConfigExport(c *check.C) {
 			continue
 		}
 		var cluster arvados.Cluster
-		c.Log(resp.Body.String())
 		err := json.Unmarshal(resp.Body.Bytes(), &cluster)
 		c.Check(err, check.IsNil)
 		c.Check(cluster.ManagementToken, check.Equals, "")
 		c.Check(cluster.SystemRootToken, check.Equals, "")
-		c.Check(cluster.Collections.BlobSigning, check.DeepEquals, true)
+		c.Check(cluster.Collections.BlobSigning, check.Equals, true)
 		c.Check(cluster.Collections.BlobSigningTTL, check.Equals, arvados.Duration(23*time.Second))
 	}
 }
@@ -167,8 +170,9 @@ func (s *HandlerSuite) TestProxyNotFound(c *check.C) {
 }
 
 func (s *HandlerSuite) TestProxyRedirect(c *check.C) {
-	s.cluster.Login.ProviderAppID = "test"
-	s.cluster.Login.ProviderAppSecret = "test"
+	s.cluster.Login.SSO.Enable = true
+	s.cluster.Login.SSO.ProviderAppID = "test"
+	s.cluster.Login.SSO.ProviderAppSecret = "test"
 	req := httptest.NewRequest("GET", "https://0.0.0.0:1/login?return_to=foo", nil)
 	resp := httptest.NewRecorder()
 	s.handler.ServeHTTP(resp, req)
@@ -182,7 +186,8 @@ func (s *HandlerSuite) TestProxyRedirect(c *check.C) {
 }
 
 func (s *HandlerSuite) TestLogoutSSO(c *check.C) {
-	s.cluster.Login.ProviderAppID = "test"
+	s.cluster.Login.SSO.Enable = true
+	s.cluster.Login.SSO.ProviderAppID = "test"
 	req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
 	resp := httptest.NewRecorder()
 	s.handler.ServeHTTP(resp, req)
@@ -197,7 +202,8 @@ func (s *HandlerSuite) TestLogoutGoogle(c *check.C) {
 		// Google login N/A
 		return
 	}
-	s.cluster.Login.GoogleClientID = "test"
+	s.cluster.Login.Google.Enable = true
+	s.cluster.Login.Google.ClientID = "test"
 	req := httptest.NewRequest("GET", "https://0.0.0.0:1/logout?return_to=https://example.com/foo", nil)
 	resp := httptest.NewRecorder()
 	s.handler.ServeHTTP(resp, req)
diff --git a/lib/controller/integration_test.go b/lib/controller/integration_test.go
index d2ae1f6fbb..3bf64771d7 100644
--- a/lib/controller/integration_test.go
+++ b/lib/controller/integration_test.go
@@ -7,9 +7,11 @@ package controller
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"io"
 	"math"
 	"net"
+	"net/http"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -84,19 +86,26 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
         Insecure: true
         Proxy: true
         ActivateUsers: true
-      z2222:
+`
+		if id != "z2222" {
+			yaml += `      z2222:
         Host: ` + hostport["z2222"] + `
         Scheme: https
         Insecure: true
         Proxy: true
         ActivateUsers: true
-      z3333:
+`
+		}
+		if id != "z3333" {
+			yaml += `      z3333:
         Host: ` + hostport["z3333"] + `
         Scheme: https
         Insecure: true
         Proxy: true
         ActivateUsers: true
 `
+		}
+
 		loader := config.NewLoader(bytes.NewBufferString(yaml), ctxlog.TestLogger(c))
 		loader.Path = "-"
 		loader.SkipLegacy = true
@@ -114,7 +123,7 @@ func (s *IntegrationSuite) SetUpSuite(c *check.C) {
 			},
 			config: *cfg,
 		}
-		s.testClusters[id].super.Start(context.Background(), &s.testClusters[id].config)
+		s.testClusters[id].super.Start(context.Background(), &s.testClusters[id].config, "-")
 	}
 	for _, tc := range s.testClusters {
 		au, ok := tc.super.WaitReady()
@@ -225,11 +234,82 @@ func (s *IntegrationSuite) TestGetCollectionByPDH(c *check.C) {
 	c.Check(coll.PortableDataHash, check.Equals, pdh)
 }
 
+// Get a token from the login cluster (z1111), use it to submit a
+// container request on z2222.
+func (s *IntegrationSuite) TestCreateContainerRequestWithFedToken(c *check.C) {
+	conn1 := s.conn("z1111")
+	rootctx1, _, _ := s.rootClients("z1111")
+	_, ac1, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
+
+	// Use ac2 to get the discovery doc with a blank token, so the
+	// SDK doesn't magically pass the z1111 token to z2222 before
+	// we're ready to start our test.
+	_, ac2, _ := s.clientsWithToken("z2222", "")
+	var dd map[string]interface{}
+	err := ac2.RequestAndDecode(&dd, "GET", "discovery/v1/apis/arvados/v1/rest", nil, nil)
+	c.Assert(err, check.IsNil)
+
+	var (
+		body bytes.Buffer
+		req  *http.Request
+		resp *http.Response
+		u    arvados.User
+		cr   arvados.ContainerRequest
+	)
+	json.NewEncoder(&body).Encode(map[string]interface{}{
+		"container_request": map[string]interface{}{
+			"command":         []string{"echo"},
+			"container_image": "d41d8cd98f00b204e9800998ecf8427e+0",
+			"cwd":             "/",
+			"output_path":     "/",
+		},
+	})
+	ac2.AuthToken = ac1.AuthToken
+
+	c.Log("...post CR with good (but not yet cached) token")
+	cr = arvados.ContainerRequest{}
+	req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
+	c.Assert(err, check.IsNil)
+	req.Header.Set("Content-Type", "application/json")
+	err = ac2.DoAndDecode(&cr, req)
+	c.Logf("err == %#v", err)
+
+	c.Log("...get user with good token")
+	u = arvados.User{}
+	req, err = http.NewRequest("GET", "https://"+ac2.APIHost+"/arvados/v1/users/current", nil)
+	c.Assert(err, check.IsNil)
+	err = ac2.DoAndDecode(&u, req)
+	c.Check(err, check.IsNil)
+	c.Check(u.UUID, check.Matches, "z1111-tpzed-.*")
+
+	c.Log("...post CR with good cached token")
+	cr = arvados.ContainerRequest{}
+	req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
+	c.Assert(err, check.IsNil)
+	req.Header.Set("Content-Type", "application/json")
+	err = ac2.DoAndDecode(&cr, req)
+	c.Check(err, check.IsNil)
+	c.Check(cr.UUID, check.Matches, "z2222-.*")
+
+	c.Log("...post with good cached token ('OAuth2 ...')")
+	cr = arvados.ContainerRequest{}
+	req, err = http.NewRequest("POST", "https://"+ac2.APIHost+"/arvados/v1/container_requests", bytes.NewReader(body.Bytes()))
+	c.Assert(err, check.IsNil)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "OAuth2 "+ac2.AuthToken)
+	resp, err = arvados.InsecureHTTPClient.Do(req)
+	if c.Check(err, check.IsNil) {
+		err = json.NewDecoder(resp.Body).Decode(&cr)
+		c.Check(cr.UUID, check.Matches, "z2222-.*")
+	}
+}
+
 // Test for bug #16263
 func (s *IntegrationSuite) TestListUsers(c *check.C) {
 	rootctx1, _, _ := s.rootClients("z1111")
 	conn1 := s.conn("z1111")
 	conn3 := s.conn("z3333")
+	userctx1, _, _ := s.userClients(rootctx1, c, conn1, "z1111", true)
 
 	// Make sure LoginCluster is properly configured
 	for cls := range s.testClusters {
@@ -239,7 +319,9 @@ func (s *IntegrationSuite) TestListUsers(c *check.C) {
 			check.Commentf("incorrect LoginCluster config on cluster %q", cls))
 	}
 	// Make sure z1111 has users with NULL usernames
-	lst, err := conn1.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+	lst, err := conn1.UserList(rootctx1, arvados.ListOptions{
+		Limit: math.MaxInt64, // check that large limit works (see #16263)
+	})
 	nullUsername := false
 	c.Assert(err, check.IsNil)
 	c.Assert(len(lst.Items), check.Not(check.Equals), 0)
@@ -249,27 +331,45 @@ func (s *IntegrationSuite) TestListUsers(c *check.C) {
 		}
 	}
 	c.Assert(nullUsername, check.Equals, true)
+
+	user1, err := conn1.UserGetCurrent(userctx1, arvados.GetOptions{})
+	c.Assert(err, check.IsNil)
+	c.Check(user1.IsActive, check.Equals, true)
+
 	// Ask for the user list on z3333 using z1111's system root token
-	_, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
-	c.Assert(err, check.IsNil, check.Commentf("getting user list: %q", err))
-}
+	lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+	c.Assert(err, check.IsNil)
+	found := false
+	for _, user := range lst.Items {
+		if user.UUID == user1.UUID {
+			c.Check(user.IsActive, check.Equals, true)
+			found = true
+			break
+		}
+	}
+	c.Check(found, check.Equals, true)
 
-// Test for bug #16263
-func (s *IntegrationSuite) TestListUsersWithMaxLimit(c *check.C) {
-	rootctx1, _, _ := s.rootClients("z1111")
-	conn3 := s.conn("z3333")
-	maxLimit := int64(math.MaxInt64)
+	// Deactivate user acct on z1111
+	_, err = conn1.UserUnsetup(rootctx1, arvados.GetOptions{UUID: user1.UUID})
+	c.Assert(err, check.IsNil)
 
-	// Make sure LoginCluster is properly configured
-	for cls := range s.testClusters {
-		c.Check(
-			s.testClusters[cls].config.Clusters[cls].Login.LoginCluster,
-			check.Equals, "z1111",
-			check.Commentf("incorrect LoginCluster config on cluster %q", cls))
+	// Get user list from z3333, check the returned z1111 user is
+	// deactivated
+	lst, err = conn3.UserList(rootctx1, arvados.ListOptions{Limit: -1})
+	c.Assert(err, check.IsNil)
+	found = false
+	for _, user := range lst.Items {
+		if user.UUID == user1.UUID {
+			c.Check(user.IsActive, check.Equals, false)
+			found = true
+			break
+		}
 	}
+	c.Check(found, check.Equals, true)
 
-	// Ask for the user list on z3333 using z1111's system root token and
-	// limit: max int64 value.
-	_, err := conn3.UserList(rootctx1, arvados.ListOptions{Limit: maxLimit})
-	c.Assert(err, check.IsNil, check.Commentf("getting user list: %q", err))
+	// Deactivated user can see is_active==false via "get current
+	// user" API
+	user1, err = conn3.UserGetCurrent(userctx1, arvados.GetOptions{})
+	c.Assert(err, check.IsNil)
+	c.Check(user1.IsActive, check.Equals, false)
 }
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index 60263455bd..4f0035edf9 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -22,11 +22,13 @@ type Conn struct {
 
 func NewConn(cluster *arvados.Cluster) *Conn {
 	railsProxy := railsproxy.NewConn(cluster)
-	return &Conn{
+	var conn Conn
+	conn = Conn{
 		cluster:         cluster,
 		railsProxy:      railsProxy,
 		loginController: chooseLoginController(cluster, railsProxy),
 	}
+	return &conn
 }
 
 func (conn *Conn) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
diff --git a/lib/controller/localdb/docker_test.go b/lib/controller/localdb/docker_test.go
new file mode 100644
index 0000000000..90c98b7d57
--- /dev/null
+++ b/lib/controller/localdb/docker_test.go
@@ -0,0 +1,68 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"io"
+	"net"
+	"strings"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	check "gopkg.in/check.v1"
+)
+
+type pgproxy struct {
+	net.Listener
+}
+
+// newPgProxy sets up a TCP proxy, listening on all interfaces, that
+// forwards all connections to the cluster's PostgreSQL server. This
+// allows the caller to run a docker container that can connect to a
+// postgresql instance that listens on the test host's loopback
+// interface.
+//
+// Caller is responsible for calling Close() on the returned pgproxy.
+func newPgProxy(c *check.C, cluster *arvados.Cluster) *pgproxy {
+	host := cluster.PostgreSQL.Connection["host"]
+	if host == "" {
+		host = "localhost"
+	}
+	port := cluster.PostgreSQL.Connection["port"]
+	if port == "" {
+		port = "5432"
+	}
+	target := net.JoinHostPort(host, port)
+
+	ln, err := net.Listen("tcp", ":")
+	c.Assert(err, check.IsNil)
+	go func() {
+		for {
+			downstream, err := ln.Accept()
+			if err != nil && strings.Contains(err.Error(), "use of closed network connection") {
+				return
+			}
+			c.Assert(err, check.IsNil)
+			go func() {
+				c.Logf("pgproxy accepted connection from %s", downstream.RemoteAddr().String())
+				defer downstream.Close()
+				upstream, err := net.Dial("tcp", target)
+				if err != nil {
+					c.Logf("net.Dial(%q): %s", target, err)
+					return
+				}
+				defer upstream.Close()
+				go io.Copy(downstream, upstream)
+				io.Copy(upstream, downstream)
+			}()
+		}
+	}()
+	c.Logf("pgproxy listening at %s", ln.Addr().String())
+	return &pgproxy{Listener: ln}
+}
+
+func (proxy *pgproxy) Port() string {
+	_, port, _ := net.SplitHostPort(proxy.Addr().String())
+	return port
+}
diff --git a/lib/controller/localdb/login.go b/lib/controller/localdb/login.go
index ae59849993..ee1ea56924 100644
--- a/lib/controller/localdb/login.go
+++ b/lib/controller/localdb/login.go
@@ -6,10 +6,18 @@ package localdb
 
 import (
 	"context"
+	"database/sql"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
+	"net/url"
+	"strings"
 
+	"git.arvados.org/arvados.git/lib/controller/rpc"
+	"git.arvados.org/arvados.git/lib/ctrlctx"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
 )
 
@@ -20,19 +28,43 @@ type loginController interface {
 }
 
 func chooseLoginController(cluster *arvados.Cluster, railsProxy *railsProxy) loginController {
-	wantGoogle := cluster.Login.GoogleClientID != ""
-	wantSSO := cluster.Login.ProviderAppID != ""
-	wantPAM := cluster.Login.PAM
+	wantGoogle := cluster.Login.Google.Enable
+	wantOpenIDConnect := cluster.Login.OpenIDConnect.Enable
+	wantSSO := cluster.Login.SSO.Enable
+	wantPAM := cluster.Login.PAM.Enable
+	wantLDAP := cluster.Login.LDAP.Enable
 	switch {
-	case wantGoogle && !wantSSO && !wantPAM:
-		return &googleLoginController{Cluster: cluster, RailsProxy: railsProxy}
-	case !wantGoogle && wantSSO && !wantPAM:
+	case wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
+		return &oidcLoginController{
+			Cluster:            cluster,
+			RailsProxy:         railsProxy,
+			Issuer:             "https://accounts.google.com",
+			ClientID:           cluster.Login.Google.ClientID,
+			ClientSecret:       cluster.Login.Google.ClientSecret,
+			UseGooglePeopleAPI: cluster.Login.Google.AlternateEmailAddresses,
+			EmailClaim:         "email",
+			EmailVerifiedClaim: "email_verified",
+		}
+	case !wantGoogle && wantOpenIDConnect && !wantSSO && !wantPAM && !wantLDAP:
+		return &oidcLoginController{
+			Cluster:            cluster,
+			RailsProxy:         railsProxy,
+			Issuer:             cluster.Login.OpenIDConnect.Issuer,
+			ClientID:           cluster.Login.OpenIDConnect.ClientID,
+			ClientSecret:       cluster.Login.OpenIDConnect.ClientSecret,
+			EmailClaim:         cluster.Login.OpenIDConnect.EmailClaim,
+			EmailVerifiedClaim: cluster.Login.OpenIDConnect.EmailVerifiedClaim,
+			UsernameClaim:      cluster.Login.OpenIDConnect.UsernameClaim,
+		}
+	case !wantGoogle && !wantOpenIDConnect && wantSSO && !wantPAM && !wantLDAP:
 		return &ssoLoginController{railsProxy}
-	case !wantGoogle && !wantSSO && wantPAM:
+	case !wantGoogle && !wantOpenIDConnect && !wantSSO && wantPAM && !wantLDAP:
 		return &pamLoginController{Cluster: cluster, RailsProxy: railsProxy}
+	case !wantGoogle && !wantOpenIDConnect && !wantSSO && !wantPAM && wantLDAP:
+		return &ldapLoginController{Cluster: cluster, RailsProxy: railsProxy}
 	default:
 		return errorLoginController{
-			error: errors.New("configuration problem: exactly one of Login.GoogleClientID, Login.ProviderAppID, or Login.PAM must be configured"),
+			error: errors.New("configuration problem: exactly one of Login.Google, Login.OpenIDConnect, Login.SSO, Login.PAM, and Login.LDAP must be enabled"),
 		}
 	}
 }
@@ -68,3 +100,47 @@ func noopLogout(cluster *arvados.Cluster, opts arvados.LogoutOptions) (arvados.L
 	}
 	return arvados.LogoutResponse{RedirectLocation: target}, nil
 }
+
+func createAPIClientAuthorization(ctx context.Context, conn *rpc.Conn, rootToken string, authinfo rpc.UserSessionAuthInfo) (resp arvados.APIClientAuthorization, err error) {
+	ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{rootToken}})
+	newsession, err := conn.UserSessionCreate(ctxRoot, rpc.UserSessionCreateOptions{
+		// Send a fake ReturnTo value instead of the caller's
+		// opts.ReturnTo. We won't follow the resulting
+		// redirect target anyway.
+		ReturnTo: ",https://none.invalid",
+		AuthInfo: authinfo,
+	})
+	if err != nil {
+		return
+	}
+	target, err := url.Parse(newsession.RedirectLocation)
+	if err != nil {
+		return
+	}
+	token := target.Query().Get("api_token")
+	tx, err := ctrlctx.CurrentTx(ctx)
+	if err != nil {
+		return
+	}
+	tokensecret := token
+	if strings.Contains(token, "/") {
+		tokenparts := strings.Split(token, "/")
+		if len(tokenparts) >= 3 {
+			tokensecret = tokenparts[2]
+		}
+	}
+	var exp sql.NullString
+	var scopes []byte
+	err = tx.QueryRowxContext(ctx, "select uuid, api_token, expires_at, scopes from api_client_authorizations where api_token=$1", tokensecret).Scan(&resp.UUID, &resp.APIToken, &exp, &scopes)
+	if err != nil {
+		return
+	}
+	resp.ExpiresAt = exp.String
+	if len(scopes) > 0 {
+		err = json.Unmarshal(scopes, &resp.Scopes)
+		if err != nil {
+			return resp, fmt.Errorf("unmarshal scopes: %s", err)
+		}
+	}
+	return
+}
diff --git a/lib/controller/localdb/login_ldap.go b/lib/controller/localdb/login_ldap.go
new file mode 100644
index 0000000000..6c430d69bb
--- /dev/null
+++ b/lib/controller/localdb/login_ldap.go
@@ -0,0 +1,152 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+
+	"git.arvados.org/arvados.git/lib/controller/rpc"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"git.arvados.org/arvados.git/sdk/go/httpserver"
+	"github.com/go-ldap/ldap"
+)
+
+type ldapLoginController struct {
+	Cluster    *arvados.Cluster
+	RailsProxy *railsProxy
+}
+
+func (ctrl *ldapLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+	return noopLogout(ctrl.Cluster, opts)
+}
+
+func (ctrl *ldapLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+	return arvados.LoginResponse{}, errors.New("interactive login is not available")
+}
+
+func (ctrl *ldapLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+	log := ctxlog.FromContext(ctx)
+	conf := ctrl.Cluster.Login.LDAP
+	errFailed := httpserver.ErrorWithStatus(fmt.Errorf("LDAP: Authentication failure (with username %q and password)", opts.Username), http.StatusUnauthorized)
+
+	if conf.SearchAttribute == "" {
+		return arvados.APIClientAuthorization{}, errors.New("config error: SearchAttribute is blank")
+	}
+	if opts.Password == "" {
+		log.WithField("username", opts.Username).Error("refusing to authenticate with empty password")
+		return arvados.APIClientAuthorization{}, errFailed
+	}
+
+	log = log.WithField("URL", conf.URL.String())
+	l, err := ldap.DialURL(conf.URL.String())
+	if err != nil {
+		log.WithError(err).Error("ldap connection failed")
+		return arvados.APIClientAuthorization{}, err
+	}
+	defer l.Close()
+
+	if conf.StartTLS {
+		var tlsconfig tls.Config
+		if conf.InsecureTLS {
+			tlsconfig.InsecureSkipVerify = true
+		} else {
+			if host, _, err := net.SplitHostPort(conf.URL.Host); err != nil {
+				// Assume SplitHostPort error means
+				// port was not specified
+				tlsconfig.ServerName = conf.URL.Host
+			} else {
+				tlsconfig.ServerName = host
+			}
+		}
+		err = l.StartTLS(&tlsconfig)
+		if err != nil {
+			log.WithError(err).Error("ldap starttls failed")
+			return arvados.APIClientAuthorization{}, err
+		}
+	}
+
+	username := opts.Username
+	if at := strings.Index(username, "@"); at >= 0 {
+		if conf.StripDomain == "*" || strings.ToLower(conf.StripDomain) == strings.ToLower(username[at+1:]) {
+			username = username[:at]
+		}
+	}
+	if conf.AppendDomain != "" && !strings.Contains(username, "@") {
+		username = username + "@" + conf.AppendDomain
+	}
+
+	if conf.SearchBindUser != "" {
+		err = l.Bind(conf.SearchBindUser, conf.SearchBindPassword)
+		if err != nil {
+			log.WithError(err).WithField("user", conf.SearchBindUser).Error("ldap authentication failed")
+			return arvados.APIClientAuthorization{}, err
+		}
+	}
+
+	search := fmt.Sprintf("(%s=%s)", ldap.EscapeFilter(conf.SearchAttribute), ldap.EscapeFilter(username))
+	if conf.SearchFilters != "" {
+		search = fmt.Sprintf("(&%s%s)", conf.SearchFilters, search)
+	}
+	log = log.WithField("search", search)
+	req := ldap.NewSearchRequest(
+		conf.SearchBase,
+		ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 1, 0, false,
+		search,
+		[]string{"DN", "givenName", "SN", conf.EmailAttribute, conf.UsernameAttribute},
+		nil)
+	resp, err := l.Search(req)
+	if ldap.IsErrorWithCode(err, ldap.LDAPResultNoResultsReturned) ||
+		ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) ||
+		(err == nil && len(resp.Entries) == 0) {
+		log.WithError(err).Info("ldap lookup returned no results")
+		return arvados.APIClientAuthorization{}, errFailed
+	} else if err != nil {
+		log.WithError(err).Error("ldap lookup failed")
+		return arvados.APIClientAuthorization{}, err
+	}
+	userdn := resp.Entries[0].DN
+	if userdn == "" {
+		log.Warn("refusing to authenticate with empty dn")
+		return arvados.APIClientAuthorization{}, errFailed
+	}
+	log = log.WithField("DN", userdn)
+
+	attrs := map[string]string{}
+	for _, attr := range resp.Entries[0].Attributes {
+		if attr == nil || len(attr.Values) == 0 {
+			continue
+		}
+		attrs[strings.ToLower(attr.Name)] = attr.Values[0]
+	}
+	log.WithField("attrs", attrs).Debug("ldap search succeeded")
+
+	// Now that we have the DN, try authenticating.
+	err = l.Bind(userdn, opts.Password)
+	if err != nil {
+		log.WithError(err).Info("ldap user authentication failed")
+		return arvados.APIClientAuthorization{}, errFailed
+	}
+	log.Debug("ldap authentication succeeded")
+
+	email := attrs[strings.ToLower(conf.EmailAttribute)]
+	if email == "" {
+		log.Errorf("ldap returned no email address in %q attribute", conf.EmailAttribute)
+		return arvados.APIClientAuthorization{}, errors.New("authentication succeeded but ldap returned no email address")
+	}
+
+	return createAPIClientAuthorization(ctx, ctrl.RailsProxy, ctrl.Cluster.SystemRootToken, rpc.UserSessionAuthInfo{
+		Email:     email,
+		FirstName: attrs["givenname"],
+		LastName:  attrs["sn"],
+		Username:  attrs[strings.ToLower(conf.UsernameAttribute)],
+	})
+}
diff --git a/lib/controller/localdb/login_ldap_docker_test.go b/lib/controller/localdb/login_ldap_docker_test.go
new file mode 100644
index 0000000000..3cbf14fe0b
--- /dev/null
+++ b/lib/controller/localdb/login_ldap_docker_test.go
@@ -0,0 +1,54 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"os"
+	"os/exec"
+	"testing"
+
+	check "gopkg.in/check.v1"
+)
+
+func haveDocker() bool {
+	_, err := exec.Command("docker", "info").CombinedOutput()
+	return err == nil
+}
+
+func (s *LDAPSuite) TestLoginLDAPViaPAM(c *check.C) {
+	if testing.Short() {
+		c.Skip("skipping docker test in short mode")
+	}
+	if !haveDocker() {
+		c.Skip("skipping docker test because docker is not available")
+	}
+	pgproxy := newPgProxy(c, s.cluster)
+	defer pgproxy.Close()
+
+	cmd := exec.Command("bash", "login_ldap_docker_test.sh")
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(), "config_method=pam", "pgport="+pgproxy.Port())
+	err := cmd.Run()
+	c.Check(err, check.IsNil)
+}
+
+func (s *LDAPSuite) TestLoginLDAPBuiltin(c *check.C) {
+	if testing.Short() {
+		c.Skip("skipping docker test in short mode")
+	}
+	if !haveDocker() {
+		c.Skip("skipping docker test because docker is not available")
+	}
+	pgproxy := newPgProxy(c, s.cluster)
+	defer pgproxy.Close()
+
+	cmd := exec.Command("bash", "login_ldap_docker_test.sh")
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	cmd.Env = append(os.Environ(), "config_method=ldap", "pgport="+pgproxy.Port())
+	err := cmd.Run()
+	c.Check(err, check.IsNil)
+}
diff --git a/lib/controller/localdb/login_pam_docker_test.sh b/lib/controller/localdb/login_ldap_docker_test.sh
similarity index 62%
rename from lib/controller/localdb/login_pam_docker_test.sh
rename to lib/controller/localdb/login_ldap_docker_test.sh
index b8f281bc2e..0225f20461 100755
--- a/lib/controller/localdb/login_pam_docker_test.sh
+++ b/lib/controller/localdb/login_ldap_docker_test.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
 
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
 # This script demonstrates using LDAP for Arvados user authentication.
 #
-# It configures pam_ldap(5) and arvados controller in a docker
-# container, with pam_ldap configured to authenticate against an
-# OpenLDAP server in a second docker container.
+# It configures arvados controller in a docker container, optionally
+# with pam_ldap(5) configured to authenticate against an OpenLDAP
+# server in a second docker container.
 #
 # After adding a "foo" user entry, it uses curl to check that the
 # Arvados controller's login endpoint accepts the "foo" account
@@ -24,6 +28,15 @@ if [[ -n ${ARVADOS_DEBUG} ]]; then
     set -x
 fi
 
+case "${config_method}" in
+    pam | ldap)
+        ;;
+    *)
+        echo >&2 "\$config_method env var must be 'pam' or 'ldap'"
+        exit 1
+        ;;
+esac
+
 hostname="$(hostname)"
 tmpdir="$(mktemp -d)"
 cleanup() {
@@ -65,6 +78,7 @@ Clusters:
       Connection:
         client_encoding: utf8
         host: ${hostname}
+        port: "${pgport}"
         dbname: arvados_test
         user: arvados
         password: insecure_arvados_test
@@ -86,15 +100,38 @@ Clusters:
         ExternalURL: http://0.0.0.0:9999/
         InternalURLs:
           "http://0.0.0.0:9999/": {}
-    Login:
-      PAM: true
-      # Without this magic PAMDefaultEmailDomain, inserted users would
-      # prevent subsequent database/reset from working (see
-      # database_controller.rb).
-      PAMDefaultEmailDomain: example.com
     SystemLogs:
       LogLevel: debug
 EOF
+case "${config_method}" in
+    pam)
+        setup_pam_ldap="apt update && DEBIAN_FRONTEND=noninteractive apt install -y ldap-utils libpam-ldap && pam-auth-update --package /usr/share/pam-configs/ldap"
+        cat >>"${tmpdir}/zzzzz.yml" <<EOF
+    Login:
+      PAM:
+        Enable: true
+        # Without this specific DefaultEmailDomain, inserted users
+        # would prevent subsequent database/reset from working (see
+        # database_controller.rb).
+        DefaultEmailDomain: example.com
+EOF
+        ;;
+    ldap)
+        setup_pam_ldap=""
+        cat >>"${tmpdir}/zzzzz.yml" <<EOF
+    Login:
+      LDAP:
+        Enable: true
+        URL: ${ldapurl}
+        StartTLS: false
+        SearchBase: dc=example,dc=org
+        SearchBindUser: cn=admin,dc=example,dc=org
+        SearchBindPassword: admin
+EOF
+            ;;
+esac
+
+cat >&2 "${tmpdir}/zzzzz.yml"
 
 cat >"${tmpdir}/pam_ldap.conf" <<EOF
 base dc=example,dc=org
@@ -113,12 +150,12 @@ cn: bar
 gidNumber: 11111
 description: "Example group 'bar'"
 
-dn: uid=foo,dc=example,dc=org
-uid: foo
-cn: foo
+dn: uid=foo-bar,dc=example,dc=org
+uid: foo-bar
+cn: "Foo Bar"
 givenName: Foo
 sn: Bar
-mail: foobar@example.org
+mail: foo-bar-baz@example.com
 objectClass: inetOrgPerson
 objectClass: posixAccount
 objectClass: top
@@ -130,11 +167,11 @@ shadowLastChange: 10701
 loginShell: /bin/bash
 uidNumber: 11111
 gidNumber: 11111
-homeDirectory: /home/foo
+homeDirectory: /home/foo-bar
 userPassword: ${passwordhash}
 EOF
 
-echo >&2 "Adding example user entry user=foo pass=secret (retrying until server comes up)"
+echo >&2 "Adding example user entry user=foo-bar pass=secret (retrying until server comes up)"
 docker run --rm --entrypoint= \
        -v "${tmpdir}/add_example_user.ldif":/add_example_user.ldif:ro \
        osixia/openldap:1.3.0 \
@@ -152,7 +189,7 @@ docker run --detach --rm --name=${ctrlctr} \
        -v "${tmpdir}/zzzzz.yml":/etc/arvados/config.yml:ro \
        -v $(realpath "${PWD}/../../.."):/arvados:ro \
        debian:10 \
-       bash -c "apt update && DEBIAN_FRONTEND=noninteractive apt install -y ldap-utils libpam-ldap && pam-auth-update --package /usr/share/pam-configs/ldap && arvados-server controller"
+       bash -c "${setup_pam_ldap:-true} && arvados-server controller"
 docker logs --follow ${ctrlctr} 2>$debug >$debug &
 ctrlhostport=$(docker port ${ctrlctr} 9999/tcp)
 
@@ -178,16 +215,42 @@ check_contains() {
     fi
 }
 
+set +x
+
 echo >&2 "Testing authentication failure"
-resp="$(curl -s --include -d username=foo -d password=nosecret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
+resp="$(set -x; curl -s --include -d username=foo-bar -d password=nosecret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
 check_contains "${resp}" "HTTP/1.1 401"
-check_contains "${resp}" '{"errors":["PAM: Authentication failure (with username \"foo\" and password)"]}'
+if [[ "${config_method}" = ldap ]]; then
+    check_contains "${resp}" '{"errors":["LDAP: Authentication failure (with username \"foo-bar\" and password)"]}'
+else
+    check_contains "${resp}" '{"errors":["PAM: Authentication failure (with username \"foo-bar\" and password)"]}'
+fi
 
 echo >&2 "Testing authentication success"
-resp="$(curl -s --include -d username=foo -d password=secret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
+resp="$(set -x; curl -s --include -d username=foo-bar -d password=secret "http://${ctrlhostport}/arvados/v1/users/authenticate" | tee $debug)"
 check_contains "${resp}" "HTTP/1.1 200"
 check_contains "${resp}" '"api_token":"'
 check_contains "${resp}" '"scopes":["all"]'
 check_contains "${resp}" '"uuid":"zzzzz-gj3su-'
 
+secret="${resp##*api_token\":\"}"
+secret="${secret%%\"*}"
+uuid="${resp##*uuid\":\"}"
+uuid="${uuid%%\"*}"
+token="v2/$uuid/$secret"
+echo >&2 "New token is ${token}"
+
+resp="$(set -x; curl -s --include -H "Authorization: Bearer ${token}" "http://${ctrlhostport}/arvados/v1/users/current" | tee $debug)"
+check_contains "${resp}" "HTTP/1.1 200"
+if [[ "${config_method}" = ldap ]]; then
+    # user fields come from LDAP attributes
+    check_contains "${resp}" '"first_name":"Foo"'
+    check_contains "${resp}" '"last_name":"Bar"'
+    check_contains "${resp}" '"username":"foobar"' # "-" removed by rails api
+    check_contains "${resp}" '"email":"foo-bar-baz@example.com"'
+else
+    # PAMDefaultEmailDomain
+    check_contains "${resp}" '"email":"foo-bar@example.com"'
+fi
+
 cleanup
diff --git a/lib/controller/localdb/login_ldap_test.go b/lib/controller/localdb/login_ldap_test.go
new file mode 100644
index 0000000000..0c94fa6c0e
--- /dev/null
+++ b/lib/controller/localdb/login_ldap_test.go
@@ -0,0 +1,156 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+	"context"
+	"encoding/json"
+	"net"
+	"net/http"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/lib/controller/railsproxy"
+	"git.arvados.org/arvados.git/lib/ctrlctx"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/auth"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/bradleypeabody/godap"
+	"github.com/jmoiron/sqlx"
+	check "gopkg.in/check.v1"
+)
+
+var _ = check.Suite(&LDAPSuite{})
+
+type LDAPSuite struct {
+	cluster *arvados.Cluster
+	ctrl    *ldapLoginController
+	ldap    *godap.LDAPServer // fake ldap server that accepts auth goodusername/goodpassword
+	db      *sqlx.DB
+
+	// transaction context
+	ctx      context.Context
+	rollback func() error
+}
+
+func (s *LDAPSuite) TearDownSuite(c *check.C) {
+	// Undo any changes/additions to the user database so they
+	// don't affect subsequent tests.
+	arvadostest.ResetEnv()
+	c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *LDAPSuite) SetUpSuite(c *check.C) {
+	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+	c.Assert(err, check.IsNil)
+	s.cluster, err = cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	s.ldap = &godap.LDAPServer{
+		Listener: ln,
+		Handlers: []godap.LDAPRequestHandler{
+			&godap.LDAPBindFuncHandler{
+				LDAPBindFunc: func(binddn string, bindpw []byte) bool {
+					return binddn == "cn=goodusername,dc=example,dc=com" && string(bindpw) == "goodpassword"
+				},
+			},
+			&godap.LDAPSimpleSearchFuncHandler{
+				LDAPSimpleSearchFunc: func(req *godap.LDAPSimpleSearchRequest) []*godap.LDAPSimpleSearchResultEntry {
+					if req.FilterAttr != "uid" || req.BaseDN != "dc=example,dc=com" {
+						return []*godap.LDAPSimpleSearchResultEntry{}
+					}
+					return []*godap.LDAPSimpleSearchResultEntry{
+						&godap.LDAPSimpleSearchResultEntry{
+							DN: "cn=" + req.FilterValue + "," + req.BaseDN,
+							Attrs: map[string]interface{}{
+								"SN":   req.FilterValue,
+								"CN":   req.FilterValue,
+								"uid":  req.FilterValue,
+								"mail": req.FilterValue + "@example.com",
+							},
+						},
+					}
+				},
+			},
+		},
+	}
+	go func() {
+		ctxlog.TestLogger(c).Print(s.ldap.Serve())
+	}()
+
+	s.cluster.Login.LDAP.Enable = true
+	err = json.Unmarshal([]byte(`"ldap://`+ln.Addr().String()+`"`), &s.cluster.Login.LDAP.URL)
+	s.cluster.Login.LDAP.StartTLS = false
+	s.cluster.Login.LDAP.SearchBindUser = "cn=goodusername,dc=example,dc=com"
+	s.cluster.Login.LDAP.SearchBindPassword = "goodpassword"
+	s.cluster.Login.LDAP.SearchBase = "dc=example,dc=com"
+	c.Assert(err, check.IsNil)
+	s.ctrl = &ldapLoginController{
+		Cluster:    s.cluster,
+		RailsProxy: railsproxy.NewConn(s.cluster),
+	}
+	s.db = arvadostest.DB(c, s.cluster)
+}
+
+func (s *LDAPSuite) SetUpTest(c *check.C) {
+	tx, err := s.db.Beginx()
+	c.Assert(err, check.IsNil)
+	s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx)
+	s.rollback = tx.Rollback
+}
+
+func (s *LDAPSuite) TearDownTest(c *check.C) {
+	if s.rollback != nil {
+		s.rollback()
+	}
+}
+
+func (s *LDAPSuite) TestLoginSuccess(c *check.C) {
+	conn := NewConn(s.cluster)
+	conn.loginController = s.ctrl
+	resp, err := conn.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+		Username: "goodusername",
+		Password: "goodpassword",
+	})
+	c.Check(err, check.IsNil)
+	c.Check(resp.APIToken, check.Not(check.Equals), "")
+	c.Check(resp.UUID, check.Matches, `zzzzz-gj3su-.*`)
+	c.Check(resp.Scopes, check.DeepEquals, []string{"all"})
+
+	ctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{"v2/" + resp.UUID + "/" + resp.APIToken}})
+	user, err := railsproxy.NewConn(s.cluster).UserGetCurrent(ctx, arvados.GetOptions{})
+	c.Check(err, check.IsNil)
+	c.Check(user.Email, check.Equals, "goodusername@example.com")
+	c.Check(user.Username, check.Equals, "goodusername")
+}
+
+func (s *LDAPSuite) TestLoginFailure(c *check.C) {
+	// search returns no results
+	s.cluster.Login.LDAP.SearchBase = "dc=example,dc=invalid"
+	resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+		Username: "goodusername",
+		Password: "goodpassword",
+	})
+	c.Check(err, check.ErrorMatches, `LDAP: Authentication failure \(with username "goodusername" and password\)`)
+	hs, ok := err.(interface{ HTTPStatus() int })
+	if c.Check(ok, check.Equals, true) {
+		c.Check(hs.HTTPStatus(), check.Equals, http.StatusUnauthorized)
+	}
+	c.Check(resp.APIToken, check.Equals, "")
+
+	// search returns result, but auth fails
+	s.cluster.Login.LDAP.SearchBase = "dc=example,dc=com"
+	resp, err = s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+		Username: "badusername",
+		Password: "badpassword",
+	})
+	c.Check(err, check.ErrorMatches, `LDAP: Authentication failure \(with username "badusername" and password\)`)
+	hs, ok = err.(interface{ HTTPStatus() int })
+	if c.Check(ok, check.Equals, true) {
+		c.Check(hs.HTTPStatus(), check.Equals, http.StatusUnauthorized)
+	}
+	c.Check(resp.APIToken, check.Equals, "")
+}
diff --git a/lib/controller/localdb/login_google.go b/lib/controller/localdb/login_oidc.go
similarity index 66%
rename from lib/controller/localdb/login_google.go
rename to lib/controller/localdb/login_oidc.go
index bf1754c158..9274d75d7c 100644
--- a/lib/controller/localdb/login_google.go
+++ b/lib/controller/localdb/login_oidc.go
@@ -30,70 +30,75 @@ import (
 	"google.golang.org/api/people/v1"
 )
 
-type googleLoginController struct {
-	Cluster    *arvados.Cluster
-	RailsProxy *railsProxy
+type oidcLoginController struct {
+	Cluster            *arvados.Cluster
+	RailsProxy         *railsProxy
+	Issuer             string // OIDC issuer URL, e.g., "https://accounts.google.com"
+	ClientID           string
+	ClientSecret       string
+	UseGooglePeopleAPI bool   // Use Google People API to look up alternate email addresses
+	EmailClaim         string // OpenID claim to use as email address; typically "email"
+	EmailVerifiedClaim string // If non-empty, ensure claim value is true before accepting EmailClaim; typically "email_verified"
+	UsernameClaim      string // If non-empty, use as preferred username
 
-	issuer            string // override OIDC issuer URL (normally https://accounts.google.com) for testing
-	peopleAPIBasePath string // override Google People API base URL (normally set by google pkg to https://people.googleapis.com/)
-	provider          *oidc.Provider
-	mu                sync.Mutex
+	// override Google People API base URL for testing purposes
+	// (normally empty, set by google pkg to
+	// https://people.googleapis.com/)
+	peopleAPIBasePath string
+
+	provider   *oidc.Provider        // initialized by setup()
+	oauth2conf *oauth2.Config        // initialized by setup()
+	verifier   *oidc.IDTokenVerifier // initialized by setup()
+	mu         sync.Mutex            // protects setup()
 }
 
-func (ctrl *googleLoginController) getProvider() (*oidc.Provider, error) {
+// Initialize ctrl.provider and ctrl.oauth2conf.
+func (ctrl *oidcLoginController) setup() error {
 	ctrl.mu.Lock()
 	defer ctrl.mu.Unlock()
-	if ctrl.provider == nil {
-		issuer := ctrl.issuer
-		if issuer == "" {
-			issuer = "https://accounts.google.com"
-		}
-		provider, err := oidc.NewProvider(context.Background(), issuer)
-		if err != nil {
-			return nil, err
-		}
-		ctrl.provider = provider
+	if ctrl.provider != nil {
+		// already set up
+		return nil
 	}
-	return ctrl.provider, nil
-}
-
-func (ctrl *googleLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
-	return noopLogout(ctrl.Cluster, opts)
-}
-
-func (ctrl *googleLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
-	provider, err := ctrl.getProvider()
+	redirURL, err := (*url.URL)(&ctrl.Cluster.Services.Controller.ExternalURL).Parse("/" + arvados.EndpointLogin.Path)
 	if err != nil {
-		return loginError(fmt.Errorf("error setting up OpenID Connect provider: %s", err))
+		return fmt.Errorf("error making redirect URL: %s", err)
 	}
-	redirURL, err := (*url.URL)(&ctrl.Cluster.Services.Controller.ExternalURL).Parse("/login")
+	provider, err := oidc.NewProvider(context.Background(), ctrl.Issuer)
 	if err != nil {
-		return loginError(fmt.Errorf("error making redirect URL: %s", err))
+		return err
 	}
-	conf := &oauth2.Config{
-		ClientID:     ctrl.Cluster.Login.GoogleClientID,
-		ClientSecret: ctrl.Cluster.Login.GoogleClientSecret,
+	ctrl.oauth2conf = &oauth2.Config{
+		ClientID:     ctrl.ClientID,
+		ClientSecret: ctrl.ClientSecret,
 		Endpoint:     provider.Endpoint(),
 		Scopes:       []string{oidc.ScopeOpenID, "profile", "email"},
 		RedirectURL:  redirURL.String(),
 	}
-	verifier := provider.Verifier(&oidc.Config{
-		ClientID: conf.ClientID,
+	ctrl.verifier = provider.Verifier(&oidc.Config{
+		ClientID: ctrl.ClientID,
 	})
+	ctrl.provider = provider
+	return nil
+}
+
+func (ctrl *oidcLoginController) Logout(ctx context.Context, opts arvados.LogoutOptions) (arvados.LogoutResponse, error) {
+	return noopLogout(ctrl.Cluster, opts)
+}
+
+func (ctrl *oidcLoginController) Login(ctx context.Context, opts arvados.LoginOptions) (arvados.LoginResponse, error) {
+	err := ctrl.setup()
+	if err != nil {
+		return loginError(fmt.Errorf("error setting up OpenID Connect provider: %s", err))
+	}
 	if opts.State == "" {
-		// Initiate Google sign-in.
+		// Initiate OIDC sign-in.
 		if opts.ReturnTo == "" {
 			return loginError(errors.New("missing return_to parameter"))
 		}
-		me := url.URL(ctrl.Cluster.Services.Controller.ExternalURL)
-		callback, err := me.Parse("/" + arvados.EndpointLogin.Path)
-		if err != nil {
-			return loginError(err)
-		}
-		conf.RedirectURL = callback.String()
 		state := ctrl.newOAuth2State([]byte(ctrl.Cluster.SystemRootToken), opts.Remote, opts.ReturnTo)
 		return arvados.LoginResponse{
-			RedirectLocation: conf.AuthCodeURL(state.String(),
+			RedirectLocation: ctrl.oauth2conf.AuthCodeURL(state.String(),
 				// prompt=select_account tells Google
 				// to show the "choose which Google
 				// account" page, even if the client
@@ -102,12 +107,12 @@ func (ctrl *googleLoginController) Login(ctx context.Context, opts arvados.Login
 				oauth2.SetAuthURLParam("prompt", "select_account")),
 		}, nil
 	} else {
-		// Callback after Google sign-in.
+		// Callback after OIDC sign-in.
 		state := ctrl.parseOAuth2State(opts.State)
 		if !state.verify([]byte(ctrl.Cluster.SystemRootToken)) {
 			return loginError(errors.New("invalid OAuth2 state"))
 		}
-		oauth2Token, err := conf.Exchange(ctx, opts.Code)
+		oauth2Token, err := ctrl.oauth2conf.Exchange(ctx, opts.Code)
 		if err != nil {
 			return loginError(fmt.Errorf("error in OAuth2 exchange: %s", err))
 		}
@@ -115,11 +120,11 @@ func (ctrl *googleLoginController) Login(ctx context.Context, opts arvados.Login
 		if !ok {
 			return loginError(errors.New("error in OAuth2 exchange: no ID token in OAuth2 token"))
 		}
-		idToken, err := verifier.Verify(ctx, rawIDToken)
+		idToken, err := ctrl.verifier.Verify(ctx, rawIDToken)
 		if err != nil {
 			return loginError(fmt.Errorf("error verifying ID token: %s", err))
 		}
-		authinfo, err := ctrl.getAuthInfo(ctx, ctrl.Cluster, conf, oauth2Token, idToken)
+		authinfo, err := ctrl.getAuthInfo(ctx, oauth2Token, idToken)
 		if err != nil {
 			return loginError(err)
 		}
@@ -131,7 +136,7 @@ func (ctrl *googleLoginController) Login(ctx context.Context, opts arvados.Login
 	}
 }
 
-func (ctrl *googleLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
+func (ctrl *oidcLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
 	return arvados.APIClientAuthorization{}, httpserver.ErrorWithStatus(errors.New("username/password authentication is not available"), http.StatusBadRequest)
 }
 
@@ -139,37 +144,38 @@ func (ctrl *googleLoginController) UserAuthenticate(ctx context.Context, opts ar
 // primary address at index 0. The provided defaultAddr is always
 // included in the returned slice, and is used as the primary if the
 // Google API does not indicate one.
-func (ctrl *googleLoginController) getAuthInfo(ctx context.Context, cluster *arvados.Cluster, conf *oauth2.Config, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
+func (ctrl *oidcLoginController) getAuthInfo(ctx context.Context, token *oauth2.Token, idToken *oidc.IDToken) (*rpc.UserSessionAuthInfo, error) {
 	var ret rpc.UserSessionAuthInfo
 	defer ctxlog.FromContext(ctx).WithField("ret", &ret).Debug("getAuthInfo returned")
 
-	var claims struct {
-		Name     string `json:"name"`
-		Email    string `json:"email"`
-		Verified bool   `json:"email_verified"`
-	}
+	var claims map[string]interface{}
 	if err := idToken.Claims(&claims); err != nil {
 		return nil, fmt.Errorf("error extracting claims from ID token: %s", err)
-	} else if claims.Verified {
+	} else if verified, _ := claims[ctrl.EmailVerifiedClaim].(bool); verified || ctrl.EmailVerifiedClaim == "" {
 		// Fall back to this info if the People API call
 		// (below) doesn't return a primary && verified email.
-		if names := strings.Fields(strings.TrimSpace(claims.Name)); len(names) > 1 {
+		name, _ := claims["name"].(string)
+		if names := strings.Fields(strings.TrimSpace(name)); len(names) > 1 {
 			ret.FirstName = strings.Join(names[0:len(names)-1], " ")
 			ret.LastName = names[len(names)-1]
 		} else {
 			ret.FirstName = names[0]
 		}
-		ret.Email = claims.Email
+		ret.Email, _ = claims[ctrl.EmailClaim].(string)
+	}
+
+	if ctrl.UsernameClaim != "" {
+		ret.Username, _ = claims[ctrl.UsernameClaim].(string)
 	}
 
-	if !ctrl.Cluster.Login.GoogleAlternateEmailAddresses {
+	if !ctrl.UseGooglePeopleAPI {
 		if ret.Email == "" {
-			return nil, fmt.Errorf("cannot log in with unverified email address %q", claims.Email)
+			return nil, fmt.Errorf("cannot log in with unverified email address %q", claims[ctrl.EmailClaim])
 		}
 		return &ret, nil
 	}
 
-	svc, err := people.NewService(ctx, option.WithTokenSource(conf.TokenSource(ctx, token)), option.WithScopes(people.UserEmailsReadScope))
+	svc, err := people.NewService(ctx, option.WithTokenSource(ctrl.oauth2conf.TokenSource(ctx, token)), option.WithScopes(people.UserEmailsReadScope))
 	if err != nil {
 		return nil, fmt.Errorf("error setting up People API: %s", err)
 	}
@@ -218,9 +224,13 @@ func (ctrl *googleLoginController) getAuthInfo(ctx context.Context, cluster *arv
 		return nil, errors.New("cannot log in without a verified email address")
 	}
 	for ae := range altEmails {
-		if ae != ret.Email {
-			ret.AlternateEmails = append(ret.AlternateEmails, ae)
-			if i := strings.Index(ae, "@"); i > 0 && strings.ToLower(ae[i+1:]) == strings.ToLower(ctrl.Cluster.Users.PreferDomainForUsername) {
+		if ae == ret.Email {
+			continue
+		}
+		ret.AlternateEmails = append(ret.AlternateEmails, ae)
+		if ret.Username == "" {
+			i := strings.Index(ae, "@")
+			if i > 0 && strings.ToLower(ae[i+1:]) == strings.ToLower(ctrl.Cluster.Users.PreferDomainForUsername) {
 				ret.Username = strings.SplitN(ae[:i], "+", 2)[0]
 			}
 		}
@@ -237,7 +247,7 @@ func loginError(sendError error) (resp arvados.LoginResponse, err error) {
 	return
 }
 
-func (ctrl *googleLoginController) newOAuth2State(key []byte, remote, returnTo string) oauth2State {
+func (ctrl *oidcLoginController) newOAuth2State(key []byte, remote, returnTo string) oauth2State {
 	s := oauth2State{
 		Time:     time.Now().Unix(),
 		Remote:   remote,
@@ -254,7 +264,7 @@ type oauth2State struct {
 	ReturnTo string // redirect target
 }
 
-func (ctrl *googleLoginController) parseOAuth2State(encoded string) (s oauth2State) {
+func (ctrl *oidcLoginController) parseOAuth2State(encoded string) (s oauth2State) {
 	// Errors are not checked. If decoding/parsing fails, the
 	// token will be rejected by verify().
 	decoded, _ := base64.RawURLEncoding.DecodeString(encoded)
diff --git a/lib/controller/localdb/login_google_test.go b/lib/controller/localdb/login_oidc_test.go
similarity index 64%
rename from lib/controller/localdb/login_google_test.go
rename to lib/controller/localdb/login_oidc_test.go
index 9e16e2e904..2ccb1fce2a 100644
--- a/lib/controller/localdb/login_google_test.go
+++ b/lib/controller/localdb/login_oidc_test.go
@@ -9,6 +9,7 @@ import (
 	"context"
 	"crypto/rand"
 	"crypto/rsa"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net/http"
@@ -34,11 +35,10 @@ func Test(t *testing.T) {
 	check.TestingT(t)
 }
 
-var _ = check.Suite(&LoginSuite{})
+var _ = check.Suite(&OIDCLoginSuite{})
 
-type LoginSuite struct {
+type OIDCLoginSuite struct {
 	cluster               *arvados.Cluster
-	ctx                   context.Context
 	localdb               *Conn
 	railsSpy              *arvadostest.Proxy
 	fakeIssuer            *httptest.Server
@@ -47,21 +47,23 @@ type LoginSuite struct {
 	issuerKey             *rsa.PrivateKey
 
 	// expected token request
-	validCode string
+	validCode         string
+	validClientID     string
+	validClientSecret string
 	// desired response from token endpoint
 	authEmail         string
 	authEmailVerified bool
 	authName          string
 }
 
-func (s *LoginSuite) TearDownSuite(c *check.C) {
+func (s *OIDCLoginSuite) TearDownSuite(c *check.C) {
 	// Undo any changes/additions to the user database so they
 	// don't affect subsequent tests.
 	arvadostest.ResetEnv()
 	c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
 }
 
-func (s *LoginSuite) SetUpTest(c *check.C) {
+func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
 	var err error
 	s.issuerKey, err = rsa.GenerateKey(rand.Reader, 2048)
 	c.Assert(err, check.IsNil)
@@ -83,20 +85,36 @@ func (s *LoginSuite) SetUpTest(c *check.C) {
 				"userinfo_endpoint":      s.fakeIssuer.URL + "/userinfo",
 			})
 		case "/token":
+			var clientID, clientSecret string
+			auth, _ := base64.StdEncoding.DecodeString(strings.TrimPrefix(req.Header.Get("Authorization"), "Basic "))
+			authsplit := strings.Split(string(auth), ":")
+			if len(authsplit) == 2 {
+				clientID, _ = url.QueryUnescape(authsplit[0])
+				clientSecret, _ = url.QueryUnescape(authsplit[1])
+			}
+			if clientID != s.validClientID || clientSecret != s.validClientSecret {
+				c.Logf("fakeIssuer: expected (%q, %q) got (%q, %q)", s.validClientID, s.validClientSecret, clientID, clientSecret)
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+
 			if req.Form.Get("code") != s.validCode || s.validCode == "" {
 				w.WriteHeader(http.StatusUnauthorized)
 				return
 			}
 			idToken, _ := json.Marshal(map[string]interface{}{
 				"iss":            s.fakeIssuer.URL,
-				"aud":            []string{"test%client$id"},
+				"aud":            []string{clientID},
 				"sub":            "fake-user-id",
-				"exp":            time.Now().UTC().Add(time.Minute).UnixNano(),
-				"iat":            time.Now().UTC().UnixNano(),
+				"exp":            time.Now().UTC().Add(time.Minute).Unix(),
+				"iat":            time.Now().UTC().Unix(),
 				"nonce":          "fake-nonce",
 				"email":          s.authEmail,
 				"email_verified": s.authEmailVerified,
 				"name":           s.authName,
+				"alt_verified":   true,                    // for custom claim tests
+				"alt_email":      "alt_email@example.com", // for custom claim tests
+				"alt_username":   "desired-username",      // for custom claim tests
 			})
 			json.NewEncoder(w).Encode(struct {
 				AccessToken  string `json:"access_token"`
@@ -145,40 +163,44 @@ func (s *LoginSuite) SetUpTest(c *check.C) {
 	s.fakePeopleAPIResponse = map[string]interface{}{}
 
 	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+	c.Assert(err, check.IsNil)
 	s.cluster, err = cfg.GetCluster("")
-	s.cluster.Login.ProviderAppID = ""
-	s.cluster.Login.ProviderAppSecret = ""
-	s.cluster.Login.GoogleClientID = "test%client$id"
-	s.cluster.Login.GoogleClientSecret = "test#client/secret"
-	s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
 	c.Assert(err, check.IsNil)
+	s.cluster.Login.SSO.Enable = false
+	s.cluster.Login.Google.Enable = true
+	s.cluster.Login.Google.ClientID = "test%client$id"
+	s.cluster.Login.Google.ClientSecret = "test#client/secret"
+	s.cluster.Users.PreferDomainForUsername = "PreferDomainForUsername.example.com"
+	s.validClientID = "test%client$id"
+	s.validClientSecret = "test#client/secret"
 
 	s.localdb = NewConn(s.cluster)
-	s.localdb.loginController.(*googleLoginController).issuer = s.fakeIssuer.URL
-	s.localdb.loginController.(*googleLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+	c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
+	s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeIssuer.URL
+	s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
 
 	s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
 	*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
 }
 
-func (s *LoginSuite) TearDownTest(c *check.C) {
+func (s *OIDCLoginSuite) TearDownTest(c *check.C) {
 	s.railsSpy.Close()
 }
 
-func (s *LoginSuite) TestGoogleLogout(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogout(c *check.C) {
 	resp, err := s.localdb.Logout(context.Background(), arvados.LogoutOptions{ReturnTo: "https://foo.example.com/bar"})
 	c.Check(err, check.IsNil)
 	c.Check(resp.RedirectLocation, check.Equals, "https://foo.example.com/bar")
 }
 
-func (s *LoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_Start_Bogus(c *check.C) {
 	resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{})
 	c.Check(err, check.IsNil)
 	c.Check(resp.RedirectLocation, check.Equals, "")
 	c.Check(resp.HTML.String(), check.Matches, `.*missing return_to parameter.*`)
 }
 
-func (s *LoginSuite) TestGoogleLogin_Start(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_Start(c *check.C) {
 	for _, remote := range []string{"", "zzzzz"} {
 		resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{Remote: remote, ReturnTo: "https://app.example.com/foo?bar"})
 		c.Check(err, check.IsNil)
@@ -188,7 +210,7 @@ func (s *LoginSuite) TestGoogleLogin_Start(c *check.C) {
 		c.Check(target.Host, check.Equals, issuerURL.Host)
 		q := target.Query()
 		c.Check(q.Get("client_id"), check.Equals, "test%client$id")
-		state := s.localdb.loginController.(*googleLoginController).parseOAuth2State(q.Get("state"))
+		state := s.localdb.loginController.(*oidcLoginController).parseOAuth2State(q.Get("state"))
 		c.Check(state.verify([]byte(s.cluster.SystemRootToken)), check.Equals, true)
 		c.Check(state.Time, check.Not(check.Equals), 0)
 		c.Check(state.Remote, check.Equals, remote)
@@ -196,7 +218,7 @@ func (s *LoginSuite) TestGoogleLogin_Start(c *check.C) {
 	}
 }
 
-func (s *LoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
 	state := s.startLogin(c)
 	resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
 		Code:  "first-try-a-bogus-code",
@@ -207,7 +229,7 @@ func (s *LoginSuite) TestGoogleLogin_InvalidCode(c *check.C) {
 	c.Check(resp.HTML.String(), check.Matches, `(?ms).*error in OAuth2 exchange.*cannot fetch token.*`)
 }
 
-func (s *LoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
 	s.startLogin(c)
 	resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
 		Code:  s.validCode,
@@ -218,16 +240,16 @@ func (s *LoginSuite) TestGoogleLogin_InvalidState(c *check.C) {
 	c.Check(resp.HTML.String(), check.Matches, `(?ms).*invalid OAuth2 state.*`)
 }
 
-func (s *LoginSuite) setupPeopleAPIError(c *check.C) {
+func (s *OIDCLoginSuite) setupPeopleAPIError(c *check.C) {
 	s.fakePeopleAPI = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		w.WriteHeader(http.StatusForbidden)
 		fmt.Fprintln(w, `Error 403: accessNotConfigured`)
 	}))
-	s.localdb.loginController.(*googleLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
+	s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakePeopleAPI.URL
 }
 
-func (s *LoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
-	s.cluster.Login.GoogleAlternateEmailAddresses = false
+func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
+	s.localdb.loginController.(*oidcLoginController).UseGooglePeopleAPI = false
 	s.authEmail = "joe.smith@primary.example.com"
 	s.setupPeopleAPIError(c)
 	state := s.startLogin(c)
@@ -240,7 +262,35 @@ func (s *LoginSuite) TestGoogleLogin_PeopleAPIDisabled(c *check.C) {
 	c.Check(authinfo.Email, check.Equals, "joe.smith@primary.example.com")
 }
 
-func (s *LoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
+func (s *OIDCLoginSuite) TestConfig(c *check.C) {
+	s.cluster.Login.Google.Enable = false
+	s.cluster.Login.OpenIDConnect.Enable = true
+	s.cluster.Login.OpenIDConnect.Issuer = "https://accounts.example.com/"
+	s.cluster.Login.OpenIDConnect.ClientID = "oidc-client-id"
+	s.cluster.Login.OpenIDConnect.ClientSecret = "oidc-client-secret"
+	localdb := NewConn(s.cluster)
+	ctrl := localdb.loginController.(*oidcLoginController)
+	c.Check(ctrl.Issuer, check.Equals, "https://accounts.example.com/")
+	c.Check(ctrl.ClientID, check.Equals, "oidc-client-id")
+	c.Check(ctrl.ClientSecret, check.Equals, "oidc-client-secret")
+	c.Check(ctrl.UseGooglePeopleAPI, check.Equals, false)
+
+	for _, enableAltEmails := range []bool{false, true} {
+		s.cluster.Login.OpenIDConnect.Enable = false
+		s.cluster.Login.Google.Enable = true
+		s.cluster.Login.Google.ClientID = "google-client-id"
+		s.cluster.Login.Google.ClientSecret = "google-client-secret"
+		s.cluster.Login.Google.AlternateEmailAddresses = enableAltEmails
+		localdb = NewConn(s.cluster)
+		ctrl = localdb.loginController.(*oidcLoginController)
+		c.Check(ctrl.Issuer, check.Equals, "https://accounts.google.com")
+		c.Check(ctrl.ClientID, check.Equals, "google-client-id")
+		c.Check(ctrl.ClientSecret, check.Equals, "google-client-secret")
+		c.Check(ctrl.UseGooglePeopleAPI, check.Equals, enableAltEmails)
+	}
+}
+
+func (s *OIDCLoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
 	s.setupPeopleAPIError(c)
 	state := s.startLogin(c)
 	resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
@@ -251,7 +301,102 @@ func (s *LoginSuite) TestGoogleLogin_PeopleAPIError(c *check.C) {
 	c.Check(resp.RedirectLocation, check.Equals, "")
 }
 
-func (s *LoginSuite) TestGoogleLogin_Success(c *check.C) {
+func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
+	s.cluster.Login.Google.Enable = false
+	s.cluster.Login.OpenIDConnect.Enable = true
+	json.Unmarshal([]byte(fmt.Sprintf("%q", s.fakeIssuer.URL)), &s.cluster.Login.OpenIDConnect.Issuer)
+	s.cluster.Login.OpenIDConnect.ClientID = "oidc#client#id"
+	s.cluster.Login.OpenIDConnect.ClientSecret = "oidc#client#secret"
+	s.validClientID = "oidc#client#id"
+	s.validClientSecret = "oidc#client#secret"
+	for _, trial := range []struct {
+		expectEmail string // "" if failure expected
+		setup       func()
+	}{
+		{
+			expectEmail: "user@oidc.example.com",
+			setup: func() {
+				c.Log("=== succeed because email_verified is false but not required")
+				s.authEmail = "user@oidc.example.com"
+				s.authEmailVerified = false
+				s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+				s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = ""
+				s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+			},
+		},
+		{
+			expectEmail: "",
+			setup: func() {
+				c.Log("=== fail because email_verified is false and required")
+				s.authEmail = "user@oidc.example.com"
+				s.authEmailVerified = false
+				s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+				s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "email_verified"
+				s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+			},
+		},
+		{
+			expectEmail: "user@oidc.example.com",
+			setup: func() {
+				c.Log("=== succeed because email_verified is false but config uses custom 'verified' claim")
+				s.authEmail = "user@oidc.example.com"
+				s.authEmailVerified = false
+				s.cluster.Login.OpenIDConnect.EmailClaim = "email"
+				s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
+				s.cluster.Login.OpenIDConnect.UsernameClaim = ""
+			},
+		},
+		{
+			expectEmail: "alt_email@example.com",
+			setup: func() {
+				c.Log("=== succeed with custom 'email' and 'email_verified' claims")
+				s.authEmail = "bad@wrong.example.com"
+				s.authEmailVerified = false
+				s.cluster.Login.OpenIDConnect.EmailClaim = "alt_email"
+				s.cluster.Login.OpenIDConnect.EmailVerifiedClaim = "alt_verified"
+				s.cluster.Login.OpenIDConnect.UsernameClaim = "alt_username"
+			},
+		},
+	} {
+		trial.setup()
+		if s.railsSpy != nil {
+			s.railsSpy.Close()
+		}
+		s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+		s.localdb = NewConn(s.cluster)
+		*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+
+		state := s.startLogin(c)
+		resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
+			Code:  s.validCode,
+			State: state,
+		})
+		c.Assert(err, check.IsNil)
+		if trial.expectEmail == "" {
+			c.Check(resp.HTML.String(), check.Matches, `(?ms).*Login error.*`)
+			c.Check(resp.RedirectLocation, check.Equals, "")
+			continue
+		}
+		c.Check(resp.HTML.String(), check.Equals, "")
+		target, err := url.Parse(resp.RedirectLocation)
+		c.Assert(err, check.IsNil)
+		token := target.Query().Get("api_token")
+		c.Check(token, check.Matches, `v2/zzzzz-gj3su-.{15}/.{32,50}`)
+		authinfo := getCallbackAuthInfo(c, s.railsSpy)
+		c.Check(authinfo.Email, check.Equals, trial.expectEmail)
+
+		switch s.cluster.Login.OpenIDConnect.UsernameClaim {
+		case "alt_username":
+			c.Check(authinfo.Username, check.Equals, "desired-username")
+		case "":
+			c.Check(authinfo.Username, check.Equals, "")
+		default:
+			c.Fail() // bad test case
+		}
+	}
+}
+
+func (s *OIDCLoginSuite) TestGoogleLogin_Success(c *check.C) {
 	state := s.startLogin(c)
 	resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{
 		Code:  s.validCode,
@@ -290,7 +435,7 @@ func (s *LoginSuite) TestGoogleLogin_Success(c *check.C) {
 	c.Check(err, check.ErrorMatches, `.*401 Unauthorized: Not logged in.*`)
 }
 
-func (s *LoginSuite) TestGoogleLogin_RealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_RealName(c *check.C) {
 	s.authEmail = "joe.smith@primary.example.com"
 	s.fakePeopleAPIResponse = map[string]interface{}{
 		"names": []map[string]interface{}{
@@ -317,7 +462,7 @@ func (s *LoginSuite) TestGoogleLogin_RealName(c *check.C) {
 	c.Check(authinfo.LastName, check.Equals, "Psmith")
 }
 
-func (s *LoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
 	s.authName = "Joe P. Smith"
 	s.authEmail = "joe.smith@primary.example.com"
 	state := s.startLogin(c)
@@ -332,7 +477,7 @@ func (s *LoginSuite) TestGoogleLogin_OIDCRealName(c *check.C) {
 }
 
 // People API returns some additional email addresses.
-func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
 	s.authEmail = "joe.smith@primary.example.com"
 	s.fakePeopleAPIResponse = map[string]interface{}{
 		"emailAddresses": []map[string]interface{}{
@@ -361,7 +506,7 @@ func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses(c *check.C) {
 }
 
 // Primary address is not the one initially returned by oidc.
-func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C) {
 	s.authEmail = "joe.smith@alternate.example.com"
 	s.fakePeopleAPIResponse = map[string]interface{}{
 		"emailAddresses": []map[string]interface{}{
@@ -390,7 +535,7 @@ func (s *LoginSuite) TestGoogleLogin_AlternateEmailAddresses_Primary(c *check.C)
 	c.Check(authinfo.Username, check.Equals, "jsmith")
 }
 
-func (s *LoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
+func (s *OIDCLoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
 	s.authEmail = "joe.smith@unverified.example.com"
 	s.authEmailVerified = false
 	s.fakePeopleAPIResponse = map[string]interface{}{
@@ -417,7 +562,7 @@ func (s *LoginSuite) TestGoogleLogin_NoPrimaryEmailAddress(c *check.C) {
 	c.Check(authinfo.Username, check.Equals, "")
 }
 
-func (s *LoginSuite) startLogin(c *check.C) (state string) {
+func (s *OIDCLoginSuite) startLogin(c *check.C) (state string) {
 	// Initiate login, but instead of following the redirect to
 	// the provider, just grab state from the redirect URL.
 	resp, err := s.localdb.Login(context.Background(), arvados.LoginOptions{ReturnTo: "https://app.example.com/foo?bar"})
@@ -429,7 +574,7 @@ func (s *LoginSuite) startLogin(c *check.C) (state string) {
 	return
 }
 
-func (s *LoginSuite) fakeToken(c *check.C, payload []byte) string {
+func (s *OIDCLoginSuite) fakeToken(c *check.C, payload []byte) string {
 	signer, err := jose.NewSigner(jose.SigningKey{Algorithm: jose.RS256, Key: s.issuerKey}, nil)
 	if err != nil {
 		c.Error(err)
diff --git a/lib/controller/localdb/login_pam.go b/lib/controller/localdb/login_pam.go
index 01dfc1379d..2447713a2c 100644
--- a/lib/controller/localdb/login_pam.go
+++ b/lib/controller/localdb/login_pam.go
@@ -9,12 +9,10 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"net/url"
 	"strings"
 
 	"git.arvados.org/arvados.git/lib/controller/rpc"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
-	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
 	"git.arvados.org/arvados.git/sdk/go/httpserver"
 	"github.com/msteinert/pam"
@@ -37,7 +35,7 @@ func (ctrl *pamLoginController) Login(ctx context.Context, opts arvados.LoginOpt
 func (ctrl *pamLoginController) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
 	errorMessage := ""
 	sentPassword := false
-	tx, err := pam.StartFunc(ctrl.Cluster.Login.PAMService, opts.Username, func(style pam.Style, message string) (string, error) {
+	tx, err := pam.StartFunc(ctrl.Cluster.Login.PAM.Service, opts.Username, func(style pam.Style, message string) (string, error) {
 		ctxlog.FromContext(ctx).Debugf("pam conversation: style=%v message=%q", style, message)
 		switch style {
 		case pam.ErrorMsg:
@@ -82,28 +80,15 @@ func (ctrl *pamLoginController) UserAuthenticate(ctx context.Context, opts arvad
 		return arvados.APIClientAuthorization{}, err
 	}
 	email := user
-	if domain := ctrl.Cluster.Login.PAMDefaultEmailDomain; domain != "" && !strings.Contains(email, "@") {
+	if domain := ctrl.Cluster.Login.PAM.DefaultEmailDomain; domain != "" && !strings.Contains(email, "@") {
 		email = email + "@" + domain
 	}
-	ctxlog.FromContext(ctx).WithFields(logrus.Fields{"user": user, "email": email}).Debug("pam authentication succeeded")
-	ctxRoot := auth.NewContext(ctx, &auth.Credentials{Tokens: []string{ctrl.Cluster.SystemRootToken}})
-	resp, err := ctrl.RailsProxy.UserSessionCreate(ctxRoot, rpc.UserSessionCreateOptions{
-		// Send a fake ReturnTo value instead of the caller's
-		// opts.ReturnTo. We won't follow the resulting
-		// redirect target anyway.
-		ReturnTo: ",https://none.invalid",
-		AuthInfo: rpc.UserSessionAuthInfo{
-			Username: user,
-			Email:    email,
-		},
+	ctxlog.FromContext(ctx).WithFields(logrus.Fields{
+		"user":  user,
+		"email": email,
+	}).Debug("pam authentication succeeded")
+	return createAPIClientAuthorization(ctx, ctrl.RailsProxy, ctrl.Cluster.SystemRootToken, rpc.UserSessionAuthInfo{
+		Username: user,
+		Email:    email,
 	})
-	if err != nil {
-		return arvados.APIClientAuthorization{}, err
-	}
-	target, err := url.Parse(resp.RedirectLocation)
-	if err != nil {
-		return arvados.APIClientAuthorization{}, err
-	}
-	token := target.Query().Get("api_token")
-	return ctrl.RailsProxy.APIClientAuthorizationCurrent(auth.NewContext(ctx, auth.NewCredentials(token)), arvados.GetOptions{})
 }
diff --git a/lib/controller/localdb/login_pam_docker_test.go b/lib/controller/localdb/login_pam_docker_test.go
deleted file mode 100644
index 8a02b2c382..0000000000
--- a/lib/controller/localdb/login_pam_docker_test.go
+++ /dev/null
@@ -1,23 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-// Skip this slow test unless invoked as "go test -tags docker".
-// +build docker
-
-package localdb
-
-import (
-	"os"
-	"os/exec"
-
-	check "gopkg.in/check.v1"
-)
-
-func (s *PamSuite) TestLoginLDAPViaPAM(c *check.C) {
-	cmd := exec.Command("bash", "login_pam_docker_test.sh")
-	cmd.Stdout = os.Stderr
-	cmd.Stderr = os.Stderr
-	err := cmd.Run()
-	c.Check(err, check.IsNil)
-}
diff --git a/lib/controller/localdb/login_pam_test.go b/lib/controller/localdb/login_pam_test.go
index 5b0e4533e1..e6b967c944 100644
--- a/lib/controller/localdb/login_pam_test.go
+++ b/lib/controller/localdb/login_pam_test.go
@@ -32,8 +32,8 @@ func (s *PamSuite) SetUpSuite(c *check.C) {
 	c.Assert(err, check.IsNil)
 	s.cluster, err = cfg.GetCluster("")
 	c.Assert(err, check.IsNil)
-	s.cluster.Login.PAM = true
-	s.cluster.Login.PAMDefaultEmailDomain = "example.com"
+	s.cluster.Login.PAM.Enable = true
+	s.cluster.Login.PAM.DefaultEmailDomain = "example.com"
 	s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
 	s.ctrl = &pamLoginController{
 		Cluster:    s.cluster,
@@ -79,6 +79,6 @@ func (s *PamSuite) TestLoginSuccess(c *check.C) {
 	c.Check(resp.Scopes, check.DeepEquals, []string{"all"})
 
 	authinfo := getCallbackAuthInfo(c, s.railsSpy)
-	c.Check(authinfo.Email, check.Equals, u+"@"+s.cluster.Login.PAMDefaultEmailDomain)
+	c.Check(authinfo.Email, check.Equals, u+"@"+s.cluster.Login.PAM.DefaultEmailDomain)
 	c.Check(authinfo.AlternateEmails, check.DeepEquals, []string(nil))
 }
diff --git a/lib/controller/proxy.go b/lib/controller/proxy.go
index 939868a17b..d7381860ea 100644
--- a/lib/controller/proxy.go
+++ b/lib/controller/proxy.go
@@ -77,9 +77,7 @@ func (p *proxy) Do(
 		Header: hdrOut,
 		Body:   reqIn.Body,
 	}).WithContext(reqIn.Context())
-
-	resp, err := client.Do(reqOut)
-	return resp, err
+	return client.Do(reqOut)
 }
 
 // Copy a response (or error) to the downstream client
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index c347e2f795..2944524344 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"strings"
 
+	"git.arvados.org/arvados.git/lib/controller/api"
 	"git.arvados.org/arvados.git/sdk/go/arvados"
 	"git.arvados.org/arvados.git/sdk/go/auth"
 	"git.arvados.org/arvados.git/sdk/go/ctxlog"
@@ -19,144 +20,152 @@ import (
 )
 
 type router struct {
-	mux *mux.Router
-	fed arvados.API
+	mux       *mux.Router
+	backend   arvados.API
+	wrapCalls func(api.RoutableFunc) api.RoutableFunc
 }
 
-func New(fed arvados.API) *router {
+// New returns a new router (which implements the http.Handler
+// interface) that serves requests by calling Arvados API methods on
+// the given backend.
+//
+// If wrapCalls is not nil, it is called once for each API method, and
+// the returned method is used in its place. This can be used to
+// install hooks before and after each API call and alter responses;
+// see localdb.WrapCallsInTransaction for an example.
+func New(backend arvados.API, wrapCalls func(api.RoutableFunc) api.RoutableFunc) *router {
 	rtr := &router{
-		mux: mux.NewRouter(),
-		fed: fed,
+		mux:       mux.NewRouter(),
+		backend:   backend,
+		wrapCalls: wrapCalls,
 	}
 	rtr.addRoutes()
 	return rtr
 }
 
-type routableFunc func(ctx context.Context, opts interface{}) (interface{}, error)
-
 func (rtr *router) addRoutes() {
 	for _, route := range []struct {
 		endpoint    arvados.APIEndpoint
 		defaultOpts func() interface{}
-		exec        routableFunc
+		exec        api.RoutableFunc
 	}{
 		{
 			arvados.EndpointConfigGet,
 			func() interface{} { return &struct{}{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ConfigGet(ctx)
+				return rtr.backend.ConfigGet(ctx)
 			},
 		},
 		{
 			arvados.EndpointLogin,
 			func() interface{} { return &arvados.LoginOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.Login(ctx, *opts.(*arvados.LoginOptions))
+				return rtr.backend.Login(ctx, *opts.(*arvados.LoginOptions))
 			},
 		},
 		{
 			arvados.EndpointLogout,
 			func() interface{} { return &arvados.LogoutOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.Logout(ctx, *opts.(*arvados.LogoutOptions))
+				return rtr.backend.Logout(ctx, *opts.(*arvados.LogoutOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionCreate,
 			func() interface{} { return &arvados.CreateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionCreate(ctx, *opts.(*arvados.CreateOptions))
+				return rtr.backend.CollectionCreate(ctx, *opts.(*arvados.CreateOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionUpdate,
 			func() interface{} { return &arvados.UpdateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionUpdate(ctx, *opts.(*arvados.UpdateOptions))
+				return rtr.backend.CollectionUpdate(ctx, *opts.(*arvados.UpdateOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionGet,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionGet(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.CollectionGet(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionList,
 			func() interface{} { return &arvados.ListOptions{Limit: -1} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionList(ctx, *opts.(*arvados.ListOptions))
+				return rtr.backend.CollectionList(ctx, *opts.(*arvados.ListOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionProvenance,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionProvenance(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.CollectionProvenance(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionUsedBy,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionUsedBy(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.CollectionUsedBy(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionDelete,
 			func() interface{} { return &arvados.DeleteOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionDelete(ctx, *opts.(*arvados.DeleteOptions))
+				return rtr.backend.CollectionDelete(ctx, *opts.(*arvados.DeleteOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionTrash,
 			func() interface{} { return &arvados.DeleteOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionTrash(ctx, *opts.(*arvados.DeleteOptions))
+				return rtr.backend.CollectionTrash(ctx, *opts.(*arvados.DeleteOptions))
 			},
 		},
 		{
 			arvados.EndpointCollectionUntrash,
 			func() interface{} { return &arvados.UntrashOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.CollectionUntrash(ctx, *opts.(*arvados.UntrashOptions))
+				return rtr.backend.CollectionUntrash(ctx, *opts.(*arvados.UntrashOptions))
 			},
 		},
 		{
 			arvados.EndpointContainerCreate,
 			func() interface{} { return &arvados.CreateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerCreate(ctx, *opts.(*arvados.CreateOptions))
+				return rtr.backend.ContainerCreate(ctx, *opts.(*arvados.CreateOptions))
 			},
 		},
 		{
 			arvados.EndpointContainerUpdate,
 			func() interface{} { return &arvados.UpdateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerUpdate(ctx, *opts.(*arvados.UpdateOptions))
+				return rtr.backend.ContainerUpdate(ctx, *opts.(*arvados.UpdateOptions))
 			},
 		},
 		{
 			arvados.EndpointContainerGet,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerGet(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.ContainerGet(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointContainerList,
 			func() interface{} { return &arvados.ListOptions{Limit: -1} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerList(ctx, *opts.(*arvados.ListOptions))
+				return rtr.backend.ContainerList(ctx, *opts.(*arvados.ListOptions))
 			},
 		},
 		{
 			arvados.EndpointContainerDelete,
 			func() interface{} { return &arvados.DeleteOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerDelete(ctx, *opts.(*arvados.DeleteOptions))
+				return rtr.backend.ContainerDelete(ctx, *opts.(*arvados.DeleteOptions))
 			},
 		},
 		{
@@ -165,7 +174,7 @@ func (rtr *router) addRoutes() {
 				return &arvados.GetOptions{Select: []string{"uuid", "state", "priority", "auth_uuid", "locked_by_uuid"}}
 			},
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerLock(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.ContainerLock(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
@@ -174,144 +183,148 @@ func (rtr *router) addRoutes() {
 				return &arvados.GetOptions{Select: []string{"uuid", "state", "priority", "auth_uuid", "locked_by_uuid"}}
 			},
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.ContainerUnlock(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointSpecimenCreate,
 			func() interface{} { return &arvados.CreateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.SpecimenCreate(ctx, *opts.(*arvados.CreateOptions))
+				return rtr.backend.SpecimenCreate(ctx, *opts.(*arvados.CreateOptions))
 			},
 		},
 		{
 			arvados.EndpointSpecimenUpdate,
 			func() interface{} { return &arvados.UpdateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.SpecimenUpdate(ctx, *opts.(*arvados.UpdateOptions))
+				return rtr.backend.SpecimenUpdate(ctx, *opts.(*arvados.UpdateOptions))
 			},
 		},
 		{
 			arvados.EndpointSpecimenGet,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.SpecimenGet(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.SpecimenGet(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointSpecimenList,
 			func() interface{} { return &arvados.ListOptions{Limit: -1} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.SpecimenList(ctx, *opts.(*arvados.ListOptions))
+				return rtr.backend.SpecimenList(ctx, *opts.(*arvados.ListOptions))
 			},
 		},
 		{
 			arvados.EndpointSpecimenDelete,
 			func() interface{} { return &arvados.DeleteOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.SpecimenDelete(ctx, *opts.(*arvados.DeleteOptions))
+				return rtr.backend.SpecimenDelete(ctx, *opts.(*arvados.DeleteOptions))
 			},
 		},
 		{
 			arvados.EndpointUserCreate,
 			func() interface{} { return &arvados.CreateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserCreate(ctx, *opts.(*arvados.CreateOptions))
+				return rtr.backend.UserCreate(ctx, *opts.(*arvados.CreateOptions))
 			},
 		},
 		{
 			arvados.EndpointUserMerge,
 			func() interface{} { return &arvados.UserMergeOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserMerge(ctx, *opts.(*arvados.UserMergeOptions))
+				return rtr.backend.UserMerge(ctx, *opts.(*arvados.UserMergeOptions))
 			},
 		},
 		{
 			arvados.EndpointUserActivate,
 			func() interface{} { return &arvados.UserActivateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserActivate(ctx, *opts.(*arvados.UserActivateOptions))
+				return rtr.backend.UserActivate(ctx, *opts.(*arvados.UserActivateOptions))
 			},
 		},
 		{
 			arvados.EndpointUserSetup,
 			func() interface{} { return &arvados.UserSetupOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserSetup(ctx, *opts.(*arvados.UserSetupOptions))
+				return rtr.backend.UserSetup(ctx, *opts.(*arvados.UserSetupOptions))
 			},
 		},
 		{
 			arvados.EndpointUserUnsetup,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserUnsetup(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.UserUnsetup(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointUserGetCurrent,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserGetCurrent(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.UserGetCurrent(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointUserGetSystem,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserGetSystem(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.UserGetSystem(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointUserGet,
 			func() interface{} { return &arvados.GetOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserGet(ctx, *opts.(*arvados.GetOptions))
+				return rtr.backend.UserGet(ctx, *opts.(*arvados.GetOptions))
 			},
 		},
 		{
 			arvados.EndpointUserUpdateUUID,
 			func() interface{} { return &arvados.UpdateUUIDOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserUpdateUUID(ctx, *opts.(*arvados.UpdateUUIDOptions))
+				return rtr.backend.UserUpdateUUID(ctx, *opts.(*arvados.UpdateUUIDOptions))
 			},
 		},
 		{
 			arvados.EndpointUserUpdate,
 			func() interface{} { return &arvados.UpdateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserUpdate(ctx, *opts.(*arvados.UpdateOptions))
+				return rtr.backend.UserUpdate(ctx, *opts.(*arvados.UpdateOptions))
 			},
 		},
 		{
 			arvados.EndpointUserList,
 			func() interface{} { return &arvados.ListOptions{Limit: -1} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserList(ctx, *opts.(*arvados.ListOptions))
+				return rtr.backend.UserList(ctx, *opts.(*arvados.ListOptions))
 			},
 		},
 		{
 			arvados.EndpointUserBatchUpdate,
 			func() interface{} { return &arvados.UserBatchUpdateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserBatchUpdate(ctx, *opts.(*arvados.UserBatchUpdateOptions))
+				return rtr.backend.UserBatchUpdate(ctx, *opts.(*arvados.UserBatchUpdateOptions))
 			},
 		},
 		{
 			arvados.EndpointUserDelete,
 			func() interface{} { return &arvados.DeleteOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserDelete(ctx, *opts.(*arvados.DeleteOptions))
+				return rtr.backend.UserDelete(ctx, *opts.(*arvados.DeleteOptions))
 			},
 		},
 		{
 			arvados.EndpointUserAuthenticate,
 			func() interface{} { return &arvados.UserAuthenticateOptions{} },
 			func(ctx context.Context, opts interface{}) (interface{}, error) {
-				return rtr.fed.UserAuthenticate(ctx, *opts.(*arvados.UserAuthenticateOptions))
+				return rtr.backend.UserAuthenticate(ctx, *opts.(*arvados.UserAuthenticateOptions))
 			},
 		},
 	} {
-		rtr.addRoute(route.endpoint, route.defaultOpts, route.exec)
+		exec := route.exec
+		if rtr.wrapCalls != nil {
+			exec = rtr.wrapCalls(exec)
+		}
+		rtr.addRoute(route.endpoint, route.defaultOpts, exec)
 	}
 	rtr.mux.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		httpserver.Errors(w, []string{"API endpoint not found"}, http.StatusNotFound)
@@ -326,7 +339,7 @@ var altMethod = map[string]string{
 	"GET":   "HEAD", // Accept HEAD at any GET route
 }
 
-func (rtr *router) addRoute(endpoint arvados.APIEndpoint, defaultOpts func() interface{}, exec routableFunc) {
+func (rtr *router) addRoute(endpoint arvados.APIEndpoint, defaultOpts func() interface{}, exec api.RoutableFunc) {
 	methods := []string{endpoint.Method}
 	if alt, ok := altMethod[endpoint.Method]; ok {
 		methods = append(methods, alt)
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 4cabe70f16..c73bc64915 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -38,8 +38,8 @@ type RouterSuite struct {
 func (s *RouterSuite) SetUpTest(c *check.C) {
 	s.stub = arvadostest.APIStub{}
 	s.rtr = &router{
-		mux: mux.NewRouter(),
-		fed: &s.stub,
+		mux:     mux.NewRouter(),
+		backend: &s.stub,
 	}
 	s.rtr.addRoutes()
 }
@@ -169,7 +169,7 @@ func (s *RouterIntegrationSuite) SetUpTest(c *check.C) {
 	cluster.TLS.Insecure = true
 	arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
 	url, _ := url.Parse("https://" + os.Getenv("ARVADOS_TEST_API_HOST"))
-	s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider))
+	s.rtr = New(rpc.NewConn("zzzzz", url, true, rpc.PassthroughTokenProvider), nil)
 }
 
 func (s *RouterIntegrationSuite) TearDownSuite(c *check.C) {
diff --git a/lib/ctrlctx/db.go b/lib/ctrlctx/db.go
new file mode 100644
index 0000000000..127be489df
--- /dev/null
+++ b/lib/ctrlctx/db.go
@@ -0,0 +1,122 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package ctrlctx
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	"git.arvados.org/arvados.git/lib/controller/api"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+)
+
+var (
+	ErrNoTransaction   = errors.New("bug: there is no transaction in this context")
+	ErrContextFinished = errors.New("refusing to start a transaction after wrapped function already returned")
+)
+
+// WrapCallsInTransactions returns a call wrapper (suitable for
+// assigning to router.router.WrapCalls) that starts a new transaction
+// for each API call, and commits only if the call succeeds.
+//
+// The wrapper calls getdb() to get a database handle before each API
+// call.
+func WrapCallsInTransactions(getdb func(context.Context) (*sqlx.DB, error)) func(api.RoutableFunc) api.RoutableFunc {
+	return func(origFunc api.RoutableFunc) api.RoutableFunc {
+		return func(ctx context.Context, opts interface{}) (_ interface{}, err error) {
+			ctx, finishtx := New(ctx, getdb)
+			defer finishtx(&err)
+			return origFunc(ctx, opts)
+		}
+	}
+}
+
+// NewWithTransaction returns a child context in which the given
+// transaction will be used by any localdb API call that needs one.
+// The caller is responsible for calling Commit or Rollback on tx.
+func NewWithTransaction(ctx context.Context, tx *sqlx.Tx) context.Context {
+	txn := &transaction{tx: tx}
+	txn.setup.Do(func() {})
+	return context.WithValue(ctx, contextKeyTransaction, txn)
+}
+
+type contextKeyT string
+
+var contextKeyTransaction = contextKeyT("transaction")
+
+type transaction struct {
+	tx    *sqlx.Tx
+	err   error
+	getdb func(context.Context) (*sqlx.DB, error)
+	setup sync.Once
+}
+
+type finishFunc func(*error)
+
+// New returns a new child context that can be used with
+// CurrentTx(). It does not open a database transaction until the
+// first call to CurrentTx().
+//
+// The caller must eventually call the returned finishtx() func to
+// commit or rollback the transaction, if any.
+//
+//	func example(ctx context.Context) (err error) {
+//		ctx, finishtx := New(ctx, dber)
+//		defer finishtx(&err)
+//		// ...
+//		tx, err := CurrentTx(ctx)
+//		if err != nil {
+//			return fmt.Errorf("example: %s", err)
+//		}
+//		return tx.ExecContext(...)
+//	}
+//
+// If *err is nil, finishtx() commits the transaction and assigns any
+// resulting error to *err.
+//
+// If *err is non-nil, finishtx() rolls back the transaction, and
+// does not modify *err.
+func New(ctx context.Context, getdb func(context.Context) (*sqlx.DB, error)) (context.Context, finishFunc) {
+	txn := &transaction{getdb: getdb}
+	return context.WithValue(ctx, contextKeyTransaction, txn), func(err *error) {
+		txn.setup.Do(func() {
+			// Using (*sync.Once)Do() prevents a future
+			// call to CurrentTx() from opening a
+			// transaction which would never get committed
+			// or rolled back. If CurrentTx() hasn't been
+			// called before now, future calls will return
+			// this error.
+			txn.err = ErrContextFinished
+		})
+		if txn.tx == nil {
+			// we never [successfully] started a transaction
+			return
+		}
+		if *err != nil {
+			ctxlog.FromContext(ctx).Debug("rollback")
+			txn.tx.Rollback()
+			return
+		}
+		*err = txn.tx.Commit()
+	}
+}
+
+func CurrentTx(ctx context.Context) (*sqlx.Tx, error) {
+	txn, ok := ctx.Value(contextKeyTransaction).(*transaction)
+	if !ok {
+		return nil, ErrNoTransaction
+	}
+	txn.setup.Do(func() {
+		if db, err := txn.getdb(ctx); err != nil {
+			txn.err = err
+		} else {
+			txn.tx, txn.err = db.Beginx()
+		}
+	})
+	return txn.tx, txn.err
+}
diff --git a/lib/ctrlctx/db_test.go b/lib/ctrlctx/db_test.go
new file mode 100644
index 0000000000..5361f13c68
--- /dev/null
+++ b/lib/ctrlctx/db_test.go
@@ -0,0 +1,87 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package ctrlctx
+
+import (
+	"context"
+	"sync"
+	"sync/atomic"
+	"testing"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+	check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+var _ = check.Suite(&DatabaseSuite{})
+
+type DatabaseSuite struct{}
+
+func (*DatabaseSuite) TestTransactionContext(c *check.C) {
+	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+	c.Assert(err, check.IsNil)
+	cluster, err := cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+
+	var getterCalled int64
+	getter := func(context.Context) (*sqlx.DB, error) {
+		atomic.AddInt64(&getterCalled, 1)
+		db, err := sqlx.Open("postgres", cluster.PostgreSQL.Connection.String())
+		c.Assert(err, check.IsNil)
+		return db, nil
+	}
+	wrapper := WrapCallsInTransactions(getter)
+	wrappedFunc := wrapper(func(ctx context.Context, opts interface{}) (interface{}, error) {
+		txes := make([]*sqlx.Tx, 20)
+		var wg sync.WaitGroup
+		for i := range txes {
+			i := i
+			wg.Add(1)
+			go func() {
+				// Concurrent calls to CurrentTx(),
+				// with different children of the same
+				// parent context, will all return the
+				// same transaction.
+				defer wg.Done()
+				ctx, cancel := context.WithCancel(ctx)
+				defer cancel()
+				tx, err := CurrentTx(ctx)
+				c.Check(err, check.IsNil)
+				txes[i] = tx
+			}()
+		}
+		wg.Wait()
+		for i := range txes[1:] {
+			c.Check(txes[i], check.Equals, txes[i+1])
+		}
+		return true, nil
+	})
+
+	ok, err := wrappedFunc(context.Background(), "blah")
+	c.Check(ok, check.Equals, true)
+	c.Check(err, check.IsNil)
+	c.Check(getterCalled, check.Equals, int64(1))
+
+	// When a wrapped func returns without calling CurrentTx(),
+	// calling CurrentTx() later shouldn't start a new
+	// transaction.
+	var savedctx context.Context
+	ok, err = wrapper(func(ctx context.Context, opts interface{}) (interface{}, error) {
+		savedctx = ctx
+		return true, nil
+	})(context.Background(), "blah")
+	c.Check(ok, check.Equals, true)
+	c.Check(err, check.IsNil)
+	tx, err := CurrentTx(savedctx)
+	c.Check(tx, check.IsNil)
+	c.Check(err, check.NotNil)
+}
diff --git a/lib/deduplicationreport/command.go b/lib/deduplicationreport/command.go
new file mode 100644
index 0000000000..1199bc0ae2
--- /dev/null
+++ b/lib/deduplicationreport/command.go
@@ -0,0 +1,43 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package deduplicationreport
+
+import (
+	"io"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+type NoPrefixFormatter struct{}
+
+func (f *NoPrefixFormatter) Format(entry *logrus.Entry) ([]byte, error) {
+	return []byte(entry.Message), nil
+}
+
+// RunCommand implements the subcommand "deduplication-report <collection> <collection> ..."
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	var err error
+	logger := ctxlog.New(stderr, "text", "info")
+	defer func() {
+		if err != nil {
+			logger.WithError(err).Error("fatal")
+		}
+	}()
+
+	logger.SetFormatter(new(NoPrefixFormatter))
+
+	loader := config.NewLoader(stdin, logger)
+	loader.SkipLegacy = true
+
+	exitcode := report(prog, args, loader, logger, stdout, stderr)
+
+	return exitcode
+}
diff --git a/lib/deduplicationreport/report.go b/lib/deduplicationreport/report.go
new file mode 100644
index 0000000000..8bb3fc4e57
--- /dev/null
+++ b/lib/deduplicationreport/report.go
@@ -0,0 +1,216 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package deduplicationreport
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"strings"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
+	"git.arvados.org/arvados.git/sdk/go/manifest"
+
+	"github.com/dustin/go-humanize"
+	"github.com/sirupsen/logrus"
+)
+
+func deDuplicate(inputs []string) (trimmed []string) {
+	seen := make(map[string]bool)
+	for _, uuid := range inputs {
+		if !seen[uuid] {
+			seen[uuid] = true
+			trimmed = append(trimmed, uuid)
+		}
+	}
+	return
+}
+
+func parseFlags(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stderr io.Writer) (exitcode int, inputs []string) {
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	flags.Usage = func() {
+		fmt.Fprintf(flags.Output(), `
+Usage:
+  %s [options ...] <collection-uuid> <collection-uuid> ...
+
+  %s [options ...] <collection-pdh>,<collection_uuid> \
+     <collection-pdh>,<collection_uuid> ...
+
+  This program analyzes the overlap in blocks used by 2 or more collections. It
+  prints a deduplication report that shows the nominal space used by the
+  collections, as well as the actual size and the amount of space that is saved
+  by Keep's deduplication.
+
+  The list of collections may be provided in two ways. A list of collection
+  uuids is sufficient. Alternatively, the PDH for each collection may also be
+  provided. This is will greatly speed up operation when the list contains
+  multiple collections with the same PDH.
+
+  Exit status will be zero if there were no errors generating the report.
+
+Example:
+
+  Use the 'arv' and 'jq' commands to get the list of the 100
+  largest collections and generate the deduplication report:
+
+  arv collection list --order 'file_size_total desc' --limit 100 | \
+    jq -r '.items[] | [.portable_data_hash,.uuid] |@csv' | \
+    tail -n+2 |sed -e 's/"//g'|tr '\n' ' ' | \
+    xargs %s
+
+Options:
+`, prog, prog, prog)
+		flags.PrintDefaults()
+	}
+	loader.SetupFlags(flags)
+	loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+	err := flags.Parse(args)
+	if err == flag.ErrHelp {
+		return 0, inputs
+	} else if err != nil {
+		return 2, inputs
+	}
+
+	inputs = flags.Args()
+
+	inputs = deDuplicate(inputs)
+
+	if len(inputs) < 1 {
+		logger.Errorf("Error: no collections provided")
+		flags.Usage()
+		return 2, inputs
+	}
+
+	lvl, err := logrus.ParseLevel(*loglevel)
+	if err != nil {
+		return 2, inputs
+	}
+	logger.SetLevel(lvl)
+	return
+}
+
+func blockList(collection arvados.Collection) (blocks map[string]int) {
+	blocks = make(map[string]int)
+	m := manifest.Manifest{Text: collection.ManifestText}
+	blockChannel := m.BlockIterWithDuplicates()
+	for b := range blockChannel {
+		blocks[b.Digest.String()] = b.Size
+	}
+	return
+}
+
+func report(prog string, args []string, loader *config.Loader, logger *logrus.Logger, stdout, stderr io.Writer) (exitcode int) {
+
+	var inputs []string
+	exitcode, inputs = parseFlags(prog, args, loader, logger, stderr)
+	if exitcode != 0 {
+		return
+	}
+
+	// Arvados Client setup
+	arv, err := arvadosclient.MakeArvadosClient()
+	if err != nil {
+		logger.Errorf("Error creating Arvados object: %s", err)
+		exitcode = 1
+		return
+	}
+
+	type Col struct {
+		FileSizeTotal int64
+		FileCount     int64
+	}
+
+	blocks := make(map[string]map[string]int)
+	pdhs := make(map[string]Col)
+	var nominalSize int64
+
+	for _, input := range inputs {
+		var uuid string
+		var pdh string
+		if strings.Contains(input, ",") {
+			// The input is in the format pdh,uuid. This will allow us to save time on duplicate pdh's
+			tmp := strings.Split(input, ",")
+			pdh = tmp[0]
+			uuid = tmp[1]
+		} else {
+			// The input must be a plain uuid
+			uuid = input
+		}
+		if !strings.Contains(uuid, "-4zz18-") {
+			logger.Errorf("Error: uuid must refer to collection object")
+			exitcode = 1
+			return
+		}
+		if _, ok := pdhs[pdh]; ok {
+			// We've processed a collection with this pdh already. Simply add its
+			// size to the totals and move on to the next one.
+			// Note that we simply trust the PDH matches the collection UUID here,
+			// in other words, we use it over the UUID. If they don't match, the report
+			// will be wrong.
+			nominalSize += pdhs[pdh].FileSizeTotal
+		} else {
+			var collection arvados.Collection
+			err = arv.Get("collections", uuid, nil, &collection)
+			if err != nil {
+				logger.Errorf("Error: unable to retrieve collection: %s", err)
+				exitcode = 1
+				return
+			}
+			blocks[uuid] = make(map[string]int)
+			blocks[uuid] = blockList(collection)
+			if pdh != "" && collection.PortableDataHash != pdh {
+				logger.Errorf("Error: the collection with UUID %s has PDH %s, but a different PDH was provided in the arguments: %s", uuid, collection.PortableDataHash, pdh)
+				exitcode = 1
+				return
+			}
+			if pdh == "" {
+				pdh = collection.PortableDataHash
+			}
+
+			col := Col{}
+			if collection.FileSizeTotal != 0 || collection.FileCount != 0 {
+				nominalSize += collection.FileSizeTotal
+				col.FileSizeTotal = collection.FileSizeTotal
+				col.FileCount = int64(collection.FileCount)
+			} else {
+				// Collections created with old Arvados versions do not always have the total file size and count cached in the collections object
+				var collSize int64
+				for _, size := range blocks[uuid] {
+					collSize += int64(size)
+				}
+				nominalSize += collSize
+				col.FileSizeTotal = collSize
+			}
+			pdhs[pdh] = col
+		}
+
+		if pdhs[pdh].FileCount != 0 {
+			fmt.Fprintf(stdout, "Collection %s: pdh %s; nominal size %d (%s); file count %d\n", uuid, pdh, pdhs[pdh].FileSizeTotal, humanize.IBytes(uint64(pdhs[pdh].FileSizeTotal)), pdhs[pdh].FileCount)
+		} else {
+			fmt.Fprintf(stdout, "Collection %s: pdh %s; nominal size %d (%s)\n", uuid, pdh, pdhs[pdh].FileSizeTotal, humanize.IBytes(uint64(pdhs[pdh].FileSizeTotal)))
+		}
+	}
+
+	var totalSize int64
+	seen := make(map[string]bool)
+	for _, v := range blocks {
+		for pdh, size := range v {
+			if !seen[pdh] {
+				seen[pdh] = true
+				totalSize += int64(size)
+			}
+		}
+	}
+	fmt.Fprintln(stdout)
+	fmt.Fprintf(stdout, "Collections:                 %15d\n", len(inputs))
+	fmt.Fprintf(stdout, "Nominal size of stored data: %15d bytes (%s)\n", nominalSize, humanize.IBytes(uint64(nominalSize)))
+	fmt.Fprintf(stdout, "Actual size of stored data:  %15d bytes (%s)\n", totalSize, humanize.IBytes(uint64(totalSize)))
+	fmt.Fprintf(stdout, "Saved by Keep deduplication: %15d bytes (%s)\n", nominalSize-totalSize, humanize.IBytes(uint64(nominalSize-totalSize)))
+
+	return exitcode
+}
diff --git a/lib/deduplicationreport/report_test.go b/lib/deduplicationreport/report_test.go
new file mode 100644
index 0000000000..a4ed46606c
--- /dev/null
+++ b/lib/deduplicationreport/report_test.go
@@ -0,0 +1,119 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package deduplicationreport
+
+import (
+	"bytes"
+	"testing"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (s *Suite) TearDownSuite(c *check.C) {
+	// Undo any changes/additions to the database so they don't affect subsequent tests.
+	arvadostest.ResetEnv()
+}
+
+func (*Suite) TestUsage(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	exitcode := Command.RunCommand("deduplicationreport.test", []string{"-log-level=debug"}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 2)
+	c.Check(stdout.String(), check.Equals, "")
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*Usage:.*`)
+}
+
+func (*Suite) TestTwoIdenticalUUIDs(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	// Run dedupreport with 2 identical uuids
+	exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooCollection, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 0)
+	c.Check(stdout.String(), check.Matches, "(?ms).*Collections:[[:space:]]+1.*")
+	c.Check(stdout.String(), check.Matches, "(?ms).*Nominal size of stored data:[[:space:]]+3 bytes \\(3 B\\).*")
+	c.Check(stdout.String(), check.Matches, "(?ms).*Actual size of stored data:[[:space:]]+3 bytes \\(3 B\\).*")
+	c.Check(stdout.String(), check.Matches, "(?ms).*Saved by Keep deduplication:[[:space:]]+0 bytes \\(0 B\\).*")
+	c.Log(stderr.String())
+}
+
+func (*Suite) TestTwoUUIDsInvalidPDH(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	// Run dedupreport with pdh,uuid where pdh does not match
+	exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooAndBarFilesInDirPDH + "," + arvadostest.FooCollection, arvadostest.FooCollection}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 1)
+	c.Check(stdout.String(), check.Equals, "")
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*Error: the collection with UUID zzzzz-4zz18-fy296fx3hot09f7 has PDH 1f4b0bc7583c2a7f9102c395f4ffc5e3\+45, but a different PDH was provided in the arguments: 870369fc72738603c2fad16664e50e2d\+58.*`)
+}
+
+func (*Suite) TestNonExistentCollection(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	// Run dedupreport with many UUIDs
+	exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooCollection, arvadostest.NonexistentCollection}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 1)
+	c.Check(stdout.String(), check.Equals, "Collection zzzzz-4zz18-fy296fx3hot09f7: pdh 1f4b0bc7583c2a7f9102c395f4ffc5e3+45; nominal size 3 (3 B)\n")
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*Error: unable to retrieve collection:.*404 Not Found.*`)
+}
+
+func (*Suite) TestManyUUIDsNoOverlap(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	// Run dedupreport with 5 UUIDs
+	exitcode := Command.RunCommand("deduplicationreport.test", []string{arvadostest.FooCollection, arvadostest.HelloWorldCollection, arvadostest.FooBarDirCollection, arvadostest.WazVersion1Collection, arvadostest.UserAgreementCollection}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 0)
+	c.Check(stdout.String(), check.Matches, "(?ms).*Collections:[[:space:]]+5.*")
+	c.Check(stdout.String(), check.Matches, "(?ms).*Nominal size of stored data:[[:space:]]+249049 bytes \\(243 KiB\\).*")
+	c.Check(stdout.String(), check.Matches, "(?ms).*Actual size of stored data:[[:space:]]+249049 bytes \\(243 KiB\\).*")
+	c.Check(stdout.String(), check.Matches, "(?ms).*Saved by Keep deduplication:[[:space:]]+0 bytes \\(0 B\\).*")
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Equals, "")
+}
+
+func (*Suite) TestTwoOverlappingCollections(c *check.C) {
+	var stdout, stderr bytes.Buffer
+	// Create two collections
+	arv := arvados.NewClientFromEnv()
+
+	var c1 arvados.Collection
+	err := arv.RequestAndDecode(&c1, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{"manifest_text": ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:foo\n"}})
+	c.Assert(err, check.Equals, nil)
+
+	var c2 arvados.Collection
+	err = arv.RequestAndDecode(&c2, "POST", "arvados/v1/collections", nil, map[string]interface{}{"collection": map[string]interface{}{"manifest_text": ". c157a79031e1c40f85931829bc5fc552+4 d3b07384d113edec49eaa6238ad5ff00+4 0:4:bar 4:4:foo\n"}})
+	c.Assert(err, check.Equals, nil)
+
+	for _, trial := range []struct {
+		field1 string
+		field2 string
+	}{
+		{
+			// Run dedupreport with 2 arguments: uuid uuid
+			field1: c1.UUID,
+			field2: c2.UUID,
+		},
+		{
+			// Run dedupreport with 2 arguments: pdh,uuid uuid
+			field1: c1.PortableDataHash + "," + c1.UUID,
+			field2: c2.UUID,
+		},
+	} {
+		exitcode := Command.RunCommand("deduplicationreport.test", []string{trial.field1, trial.field2}, &bytes.Buffer{}, &stdout, &stderr)
+		c.Check(exitcode, check.Equals, 0)
+		c.Check(stdout.String(), check.Matches, "(?ms).*Nominal size of stored data:[[:space:]]+12 bytes \\(12 B\\).*")
+		c.Check(stdout.String(), check.Matches, "(?ms).*Actual size of stored data:[[:space:]]+8 bytes \\(8 B\\).*")
+		c.Check(stdout.String(), check.Matches, "(?ms).*Saved by Keep deduplication:[[:space:]]+4 bytes \\(4 B\\).*")
+		c.Log(stderr.String())
+		c.Check(stderr.String(), check.Equals, "")
+	}
+}
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 4e1dc73746..ba57c20c35 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -311,19 +311,12 @@ rm ${zip}
 			}
 			defer func() {
 				cmd.Process.Signal(syscall.SIGTERM)
-				logger.Infof("sent SIGTERM; waiting for postgres to shut down")
+				logger.Info("sent SIGTERM; waiting for postgres to shut down")
 				cmd.Wait()
 			}()
-			for deadline := time.Now().Add(10 * time.Second); ; {
-				output, err2 := exec.Command("pg_isready").CombinedOutput()
-				if err2 == nil {
-					break
-				} else if time.Now().After(deadline) {
-					err = fmt.Errorf("timed out waiting for pg_isready (%q)", output)
-					return 1
-				} else {
-					time.Sleep(time.Second)
-				}
+			err = waitPostgreSQLReady()
+			if err != nil {
+				return 1
 			}
 		}
 
@@ -334,6 +327,51 @@ rm ${zip}
 			// might never have been run.
 		}
 
+		var needcoll []string
+		// If the en_US.UTF-8 locale wasn't installed when
+		// postgresql initdb ran, it needs to be added
+		// explicitly before we can use it in our test suite.
+		for _, collname := range []string{"en_US", "en_US.UTF-8"} {
+			cmd := exec.Command("sudo", "-u", "postgres", "psql", "-t", "-c", "SELECT 1 FROM pg_catalog.pg_collation WHERE collname='"+collname+"' AND collcollate IN ('en_US.UTF-8', 'en_US.utf8')")
+			cmd.Dir = "/"
+			out, err2 := cmd.CombinedOutput()
+			if err != nil {
+				err = fmt.Errorf("error while checking postgresql collations: %s", err2)
+				return 1
+			}
+			if strings.Contains(string(out), "1") {
+				logger.Infof("postgresql supports collation %s", collname)
+			} else {
+				needcoll = append(needcoll, collname)
+			}
+		}
+		if len(needcoll) > 0 && os.Getpid() != 1 {
+			// In order for the CREATE COLLATION statement
+			// below to work, the locale must have existed
+			// when PostgreSQL started up. If we're
+			// running as init, we must have started
+			// PostgreSQL ourselves after installing the
+			// locales. Otherwise, it might need a
+			// restart, so we attempt to restart it with
+			// systemd.
+			if err = runBash(`sudo systemctl restart postgresql`, stdout, stderr); err != nil {
+				logger.Warn("`systemctl restart postgresql` failed; hoping postgresql does not need to be restarted")
+			} else if err = waitPostgreSQLReady(); err != nil {
+				return 1
+			}
+		}
+		for _, collname := range needcoll {
+			cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "CREATE COLLATION \""+collname+"\" (LOCALE = \"en_US.UTF-8\")")
+			cmd.Stdout = stdout
+			cmd.Stderr = stderr
+			cmd.Dir = "/"
+			err = cmd.Run()
+			if err != nil {
+				err = fmt.Errorf("error adding postgresql collation %s: %s", collname, err)
+				return 1
+			}
+		}
+
 		withstuff := "WITH LOGIN SUPERUSER ENCRYPTED PASSWORD " + pq.QuoteLiteral(devtestDatabasePassword)
 		cmd := exec.Command("sudo", "-u", "postgres", "psql", "-c", "ALTER ROLE arvados "+withstuff)
 		cmd.Dir = "/"
@@ -408,6 +446,19 @@ func identifyOS() (osversion, error) {
 	return osv, nil
 }
 
+func waitPostgreSQLReady() error {
+	for deadline := time.Now().Add(10 * time.Second); ; {
+		output, err := exec.Command("pg_isready").CombinedOutput()
+		if err == nil {
+			return nil
+		} else if time.Now().After(deadline) {
+			return fmt.Errorf("timed out waiting for pg_isready (%q)", output)
+		} else {
+			time.Sleep(time.Second)
+		}
+	}
+}
+
 func runBash(script string, stdout, stderr io.Writer) error {
 	cmd := exec.Command("bash", "-")
 	cmd.Stdin = bytes.NewBufferString("set -ex -o pipefail\n" + script)
diff --git a/lib/pam/.gitignore b/lib/pam/.gitignore
new file mode 100644
index 0000000000..8d44d630d7
--- /dev/null
+++ b/lib/pam/.gitignore
@@ -0,0 +1,2 @@
+pam_arvados.h
+pam_arvados.so
diff --git a/lib/pam/README b/lib/pam/README
new file mode 100644
index 0000000000..8c5e10ed60
--- /dev/null
+++ b/lib/pam/README
@@ -0,0 +1,18 @@
+For configuration advice, please refer to https://doc.arvados.org/install/install-webshell.html
+
+Usage (in pam config):
+
+    pam_arvados.so arvados_api_host my_vm_hostname ["insecure"] ["debug"]
+
+pam_arvados.so passes authentication if (according to
+arvados_api_host) the supplied PAM token belongs to an Arvados user
+who is allowed to log in to my_vm_host_name with the supplied PAM
+username.
+
+If my_vm_hostname is omitted or "-", the current hostname is used.
+
+"insecure" -- continue even if the TLS certificate presented by
+arvados_api_host fails verification.
+
+"debug" -- enable debug-level log messages in syslog and (when not in
+"silent" mode) on the calling application's stderr.
diff --git a/lib/pam/docker_test.go b/lib/pam/docker_test.go
new file mode 100644
index 0000000000..fa16b313be
--- /dev/null
+++ b/lib/pam/docker_test.go
@@ -0,0 +1,173 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+	"bytes"
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/http/httputil"
+	"net/url"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"gopkg.in/check.v1"
+)
+
+type DockerSuite struct {
+	tmpdir   string
+	hostip   string
+	proxyln  net.Listener
+	proxysrv *http.Server
+}
+
+var _ = check.Suite(&DockerSuite{})
+
+func Test(t *testing.T) { check.TestingT(t) }
+
+func (s *DockerSuite) SetUpSuite(c *check.C) {
+	if testing.Short() {
+		c.Skip("skipping docker tests in short mode")
+	} else if _, err := exec.Command("docker", "info").CombinedOutput(); err != nil {
+		c.Skip("skipping docker tests because docker is not available")
+	}
+
+	s.tmpdir = c.MkDir()
+
+	// The integration-testing controller listens on the loopback
+	// interface, so it won't be reachable directly from the
+	// docker container -- so here we run a proxy on 0.0.0.0 for
+	// the duration of the test.
+	hostips, err := exec.Command("hostname", "-I").Output()
+	c.Assert(err, check.IsNil)
+	s.hostip = strings.Split(strings.Trim(string(hostips), "\n"), " ")[0]
+	ln, err := net.Listen("tcp", s.hostip+":0")
+	c.Assert(err, check.IsNil)
+	s.proxyln = ln
+	proxy := httputil.NewSingleHostReverseProxy(&url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")})
+	proxy.Transport = &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+	}
+	s.proxysrv = &http.Server{Handler: proxy}
+	go s.proxysrv.ServeTLS(ln, "../../services/api/tmp/self-signed.pem", "../../services/api/tmp/self-signed.key")
+
+	// Build a pam module to install & configure in the docker
+	// container.
+	cmd := exec.Command("go", "build", "-buildmode=c-shared", "-o", s.tmpdir+"/pam_arvados.so")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	c.Assert(err, check.IsNil)
+
+	// Build the testclient program that will (from inside the
+	// docker container) configure the system to use the above PAM
+	// config, and then try authentication.
+	cmd = exec.Command("go", "build", "-o", s.tmpdir+"/testclient", "./testclient.go")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	c.Assert(err, check.IsNil)
+}
+
+func (s *DockerSuite) TearDownSuite(c *check.C) {
+	if s.proxysrv != nil {
+		s.proxysrv.Close()
+	}
+	if s.proxyln != nil {
+		s.proxyln.Close()
+	}
+}
+
+func (s *DockerSuite) SetUpTest(c *check.C) {
+	// Write a PAM config file that uses our proxy as
+	// ARVADOS_API_HOST.
+	proxyhost := s.proxyln.Addr().String()
+	confdata := fmt.Sprintf(`Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+	[success=end default=ignore]	/usr/lib/pam_arvados.so %s testvm2.shell insecure
+Auth-Initial:
+	[success=end default=ignore]	/usr/lib/pam_arvados.so %s testvm2.shell insecure
+`, proxyhost, proxyhost)
+	err := ioutil.WriteFile(s.tmpdir+"/conffile", []byte(confdata), 0755)
+	c.Assert(err, check.IsNil)
+}
+
+func (s *DockerSuite) runTestClient(c *check.C, args ...string) (stdout, stderr *bytes.Buffer, err error) {
+
+	cmd := exec.Command("docker", append([]string{
+		"run", "--rm",
+		"--hostname", "testvm2.shell",
+		"--add-host", "zzzzz.arvadosapi.com:" + s.hostip,
+		"-v", s.tmpdir + "/pam_arvados.so:/usr/lib/pam_arvados.so:ro",
+		"-v", s.tmpdir + "/conffile:/usr/share/pam-configs/arvados:ro",
+		"-v", s.tmpdir + "/testclient:/testclient:ro",
+		"debian:buster",
+		"/testclient"}, args...)...)
+	stdout = &bytes.Buffer{}
+	stderr = &bytes.Buffer{}
+	cmd.Stdout = stdout
+	cmd.Stderr = stderr
+	err = cmd.Run()
+	return
+}
+
+func (s *DockerSuite) TestSuccess(c *check.C) {
+	stdout, stderr, err := s.runTestClient(c, "try", "active", arvadostest.ActiveTokenV2)
+	c.Check(err, check.IsNil)
+	c.Logf("%s", stderr.String())
+	c.Check(stdout.String(), check.Equals, "")
+	c.Check(stderr.String(), check.Matches, `(?ms).*authentication succeeded.*`)
+}
+
+func (s *DockerSuite) TestFailure(c *check.C) {
+	for _, trial := range []struct {
+		label    string
+		username string
+		token    string
+	}{
+		{"bad token", "active", arvadostest.ActiveTokenV2 + "badtoken"},
+		{"empty token", "active", ""},
+		{"empty username", "", arvadostest.ActiveTokenV2},
+		{"wrong username", "wrongusername", arvadostest.ActiveTokenV2},
+	} {
+		c.Logf("trial: %s", trial.label)
+		stdout, stderr, err := s.runTestClient(c, "try", trial.username, trial.token)
+		c.Logf("%s", stderr.String())
+		c.Check(err, check.NotNil)
+		c.Check(stdout.String(), check.Equals, "")
+		c.Check(stderr.String(), check.Matches, `(?ms).*authentication failed.*`)
+	}
+}
+
+func (s *DockerSuite) TestDefaultHostname(c *check.C) {
+	confdata := fmt.Sprintf(`Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+	[success=end default=ignore]	/usr/lib/pam_arvados.so %s - insecure debug
+Auth-Initial:
+	[success=end default=ignore]	/usr/lib/pam_arvados.so %s - insecure debug
+`, s.proxyln.Addr().String(), s.proxyln.Addr().String())
+	err := ioutil.WriteFile(s.tmpdir+"/conffile", []byte(confdata), 0755)
+	c.Assert(err, check.IsNil)
+
+	stdout, stderr, err := s.runTestClient(c, "try", "active", arvadostest.ActiveTokenV2)
+	c.Check(err, check.IsNil)
+	c.Logf("%s", stderr.String())
+	c.Check(stdout.String(), check.Equals, "")
+	c.Check(stderr.String(), check.Matches, `(?ms).*authentication succeeded.*`)
+}
diff --git a/lib/pam/fpm-info.sh b/lib/pam/fpm-info.sh
new file mode 100644
index 0000000000..3366b8e79a
--- /dev/null
+++ b/lib/pam/fpm-info.sh
@@ -0,0 +1,5 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+fpm_depends+=(ca-certificates)
diff --git a/lib/pam/pam-configs-arvados b/lib/pam/pam-configs-arvados
new file mode 100644
index 0000000000..37ed4b86ab
--- /dev/null
+++ b/lib/pam/pam-configs-arvados
@@ -0,0 +1,19 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# This file is packaged as /usr/share/pam-configs/arvados-go; see build/run-library.sh
+
+# 1. Run `pam-auth-update` and choose Arvados authentication
+# 2. In /etc/pam.d/common-auth, change "api.example" to your ARVADOS_API_HOST
+# 3. In /etc/pam.d/common-auth, change "shell.example" to this host's hostname
+#    (as it appears in the Arvados virtual_machines list)
+
+Name: Arvados authentication
+Default: yes
+Priority: 256
+Auth-Type: Primary
+Auth:
+	[success=end default=ignore]	/usr/lib/pam_arvados.so api.example shell.example
+Auth-Initial:
+	[success=end default=ignore]	/usr/lib/pam_arvados.so api.example shell.example
diff --git a/lib/pam/pam_arvados.go b/lib/pam/pam_arvados.go
new file mode 100644
index 0000000000..34b9080536
--- /dev/null
+++ b/lib/pam/pam_arvados.go
@@ -0,0 +1,185 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// To enable, add an entry in /etc/pam.d/common-auth where pam_unix.so
+// would normally be. Examples:
+//
+// auth [success=1 default=ignore] /usr/lib/pam_arvados.so zzzzz.arvadosapi.com vmhostname.example
+// auth [success=1 default=ignore] /usr/lib/pam_arvados.so zzzzz.arvadosapi.com vmhostname.example insecure debug
+//
+// Replace zzzzz.arvadosapi.com with your controller host or
+// host:port.
+//
+// Replace vmhostname.example with the VM's name as it appears in the
+// Arvados virtual_machine object.
+//
+// Use "insecure" if your API server certificate does not pass name
+// verification.
+//
+// Use "debug" to enable debug log messages.
+
+package main
+
+import (
+	"io/ioutil"
+	"log/syslog"
+	"os"
+
+	"context"
+	"errors"
+	"fmt"
+	"runtime"
+	"syscall"
+	"time"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"github.com/sirupsen/logrus"
+	lSyslog "github.com/sirupsen/logrus/hooks/syslog"
+	"golang.org/x/sys/unix"
+)
+
+/*
+#cgo LDFLAGS: -lpam -fPIC
+#include <security/pam_ext.h>
+char *stringindex(char** a, int i);
+const char *get_user(pam_handle_t *pamh);
+const char *get_authtoken(pam_handle_t *pamh);
+*/
+import "C"
+
+func main() {}
+
+func init() {
+	if err := unix.Prctl(syscall.PR_SET_DUMPABLE, 0, 0, 0, 0); err != nil {
+		newLogger(false).WithError(err).Warn("unable to disable ptrace")
+	}
+}
+
+//export pam_sm_setcred
+func pam_sm_setcred(pamh *C.pam_handle_t, flags, cArgc C.int, cArgv **C.char) C.int {
+	return C.PAM_IGNORE
+}
+
+//export pam_sm_authenticate
+func pam_sm_authenticate(pamh *C.pam_handle_t, flags, cArgc C.int, cArgv **C.char) C.int {
+	runtime.GOMAXPROCS(1)
+	logger := newLogger(flags&C.PAM_SILENT == 0)
+	cUsername := C.get_user(pamh)
+	if cUsername == nil {
+		return C.PAM_USER_UNKNOWN
+	}
+
+	cToken := C.get_authtoken(pamh)
+	if cToken == nil {
+		return C.PAM_AUTH_ERR
+	}
+
+	argv := make([]string, cArgc)
+	for i := 0; i < int(cArgc); i++ {
+		argv[i] = C.GoString(C.stringindex(cArgv, C.int(i)))
+	}
+
+	err := authenticate(logger, C.GoString(cUsername), C.GoString(cToken), argv)
+	if err != nil {
+		logger.WithError(err).Error("authentication failed")
+		return C.PAM_AUTH_ERR
+	}
+	return C.PAM_SUCCESS
+}
+
+func authenticate(logger *logrus.Logger, username, token string, argv []string) error {
+	hostname := ""
+	apiHost := ""
+	insecure := false
+	for idx, arg := range argv {
+		if idx == 0 {
+			apiHost = arg
+		} else if idx == 1 {
+			hostname = arg
+		} else if arg == "insecure" {
+			insecure = true
+		} else if arg == "debug" {
+			logger.SetLevel(logrus.DebugLevel)
+		} else {
+			logger.Warnf("unkown option: %s\n", arg)
+		}
+	}
+	if hostname == "" || hostname == "-" {
+		h, err := os.Hostname()
+		if err != nil {
+			logger.WithError(err).Warnf("cannot get hostname -- try using an explicit hostname in pam config")
+			return fmt.Errorf("cannot get hostname: %w", err)
+		}
+		hostname = h
+	}
+	logger.Debugf("username=%q arvados_api_host=%q hostname=%q insecure=%t", username, apiHost, hostname, insecure)
+	if apiHost == "" {
+		logger.Warnf("cannot authenticate: config error: arvados_api_host and hostname must be non-empty")
+		return errors.New("config error")
+	}
+	arv := &arvados.Client{
+		Scheme:    "https",
+		APIHost:   apiHost,
+		AuthToken: token,
+		Insecure:  insecure,
+	}
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute))
+	defer cancel()
+	var vms arvados.VirtualMachineList
+	err := arv.RequestAndDecodeContext(ctx, &vms, "GET", "arvados/v1/virtual_machines", nil, arvados.ListOptions{
+		Limit: 2,
+		Filters: []arvados.Filter{
+			{"hostname", "=", hostname},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if len(vms.Items) == 0 {
+		// It's possible there is no VM entry for the
+		// configured hostname, but typically this just means
+		// the user does not have permission to see (let alone
+		// log in to) this VM.
+		return errors.New("permission denied")
+	} else if len(vms.Items) > 1 {
+		return fmt.Errorf("multiple results for hostname %q", hostname)
+	} else if vms.Items[0].Hostname != hostname {
+		return fmt.Errorf("looked up hostname %q but controller returned record with hostname %q", hostname, vms.Items[0].Hostname)
+	}
+	var user arvados.User
+	err = arv.RequestAndDecodeContext(ctx, &user, "GET", "arvados/v1/users/current", nil, nil)
+	if err != nil {
+		return err
+	}
+	var links arvados.LinkList
+	err = arv.RequestAndDecodeContext(ctx, &links, "GET", "arvados/v1/links", nil, arvados.ListOptions{
+		Limit: 1,
+		Filters: []arvados.Filter{
+			{"link_class", "=", "permission"},
+			{"name", "=", "can_login"},
+			{"tail_uuid", "=", user.UUID},
+			{"head_uuid", "=", vms.Items[0].UUID},
+			{"properties.username", "=", username},
+		},
+	})
+	if err != nil {
+		return err
+	}
+	if len(links.Items) < 1 || links.Items[0].Properties["username"] != username {
+		return errors.New("permission denied")
+	}
+	logger.Debugf("permission granted based on link with UUID %s", links.Items[0].UUID)
+	return nil
+}
+
+func newLogger(stderr bool) *logrus.Logger {
+	logger := logrus.New()
+	if !stderr {
+		logger.Out = ioutil.Discard
+	}
+	if hook, err := lSyslog.NewSyslogHook("udp", "localhost:514", syslog.LOG_AUTH|syslog.LOG_INFO, "pam_arvados"); err != nil {
+		logger.Hooks.Add(hook)
+	}
+	return logger
+}
diff --git a/lib/pam/pam_c.go b/lib/pam/pam_c.go
new file mode 100644
index 0000000000..4bf975b22c
--- /dev/null
+++ b/lib/pam/pam_c.go
@@ -0,0 +1,24 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+/*
+#cgo LDFLAGS: -lpam -fPIC
+#include <security/pam_ext.h>
+char *stringindex(char** a, int i) { return a[i]; }
+const char *get_user(pam_handle_t *pamh) {
+  const char *user;
+  if (pam_get_item(pamh, PAM_USER, (const void**)&user) != PAM_SUCCESS)
+    return NULL;
+  return user;
+}
+const char *get_authtoken(pam_handle_t *pamh) {
+  const char *token;
+  if (pam_get_authtok(pamh, PAM_AUTHTOK, &token, NULL) != PAM_SUCCESS)
+    return NULL;
+  return token;
+}
+*/
+import "C"
diff --git a/lib/pam/testclient.go b/lib/pam/testclient.go
new file mode 100644
index 0000000000..3e92cac444
--- /dev/null
+++ b/lib/pam/testclient.go
@@ -0,0 +1,83 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// +build never
+
+// This file is compiled by docker_test.go to build a test client.
+// It's not part of the pam module itself.
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+
+	"github.com/msteinert/pam"
+	"github.com/sirupsen/logrus"
+)
+
+func main() {
+	if len(os.Args) != 4 || os.Args[1] != "try" {
+		logrus.Print("usage: testclient try 'username' 'password'")
+		os.Exit(1)
+	}
+	username := os.Args[2]
+	password := os.Args[3]
+
+	// Configure PAM to use arvados token auth by default.
+	cmd := exec.Command("pam-auth-update", "--force", "arvados", "--remove", "unix")
+	cmd.Env = append([]string{"DEBIAN_FRONTEND=noninteractive"}, os.Environ()...)
+	cmd.Stdin = nil
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		logrus.WithError(err).Error("pam-auth-update failed")
+		os.Exit(1)
+	}
+
+	// Check that pam-auth-update actually added arvados config.
+	cmd = exec.Command("grep", "-Hn", "arvados", "/etc/pam.d/common-auth")
+	cmd.Stdout = os.Stderr
+	cmd.Stderr = os.Stderr
+	err = cmd.Run()
+	if err != nil {
+		panic(err)
+	}
+
+	logrus.Debugf("starting pam: username=%q password=%q", username, password)
+
+	sentPassword := false
+	errorMessage := ""
+	tx, err := pam.StartFunc("default", username, func(style pam.Style, message string) (string, error) {
+		logrus.Debugf("pam conversation: style=%v message=%q", style, message)
+		switch style {
+		case pam.ErrorMsg:
+			logrus.WithField("Message", message).Info("pam.ErrorMsg")
+			errorMessage = message
+			return "", nil
+		case pam.TextInfo:
+			logrus.WithField("Message", message).Info("pam.TextInfo")
+			errorMessage = message
+			return "", nil
+		case pam.PromptEchoOn, pam.PromptEchoOff:
+			sentPassword = true
+			return password, nil
+		default:
+			return "", fmt.Errorf("unrecognized message style %d", style)
+		}
+	})
+	if err != nil {
+		logrus.WithError(err).Print("StartFunc failed")
+		os.Exit(1)
+	}
+	err = tx.Authenticate(pam.DisallowNullAuthtok)
+	if err != nil {
+		err = fmt.Errorf("PAM: %s (message = %q)", err, errorMessage)
+		logrus.WithError(err).Print("authentication failed")
+		os.Exit(1)
+	}
+	logrus.Print("authentication succeeded")
+}
diff --git a/lib/recovercollection/cmd.go b/lib/recovercollection/cmd.go
new file mode 100644
index 0000000000..da466c31ca
--- /dev/null
+++ b/lib/recovercollection/cmd.go
@@ -0,0 +1,381 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package recovercollection
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"strings"
+	"sync"
+	"time"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"github.com/sirupsen/logrus"
+)
+
+var Command command
+
+type command struct{}
+
+func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+	var err error
+	logger := ctxlog.New(stderr, "text", "info")
+	defer func() {
+		if err != nil {
+			logger.WithError(err).Error("fatal")
+		}
+		logger.Info("exiting")
+	}()
+
+	loader := config.NewLoader(stdin, logger)
+	loader.SkipLegacy = true
+
+	flags := flag.NewFlagSet("", flag.ContinueOnError)
+	flags.SetOutput(stderr)
+	flags.Usage = func() {
+		fmt.Fprintf(flags.Output(), `Usage:
+	%s [options ...] { /path/to/manifest.txt | log-or-collection-uuid } [...]
+
+	This program recovers deleted collections. Recovery is
+	possible when the collection's manifest is still available and
+	all of its data blocks are still available or recoverable
+	(e.g., garbage collection is not enabled, the blocks are too
+	new for garbage collection, the blocks are referenced by other
+	collections, or the blocks have been trashed but not yet
+	deleted).
+
+	There are multiple ways to specify a collection to recover:
+
+        * Path to a local file containing a manifest with the desired
+	  data
+
+	* UUID of an Arvados log entry, typically a "delete" or
+	  "update" event, whose "old attributes" have a manifest with
+	  the desired data
+
+	* UUID of an Arvados collection whose most recent log entry,
+          typically a "delete" or "update" event, has the desired
+          data in its "old attributes"
+
+	For each provided collection manifest, once all data blocks
+	are recovered/protected from garbage collection, a new
+	collection is saved and its UUID is printed on stdout.
+
+	Restored collections will belong to the system (root) user.
+
+	Exit status will be zero if recovery is successful, i.e., a
+	collection is saved for each provided manifest.
+Options:
+`, prog)
+		flags.PrintDefaults()
+	}
+	loader.SetupFlags(flags)
+	loglevel := flags.String("log-level", "info", "logging level (debug, info, ...)")
+	err = flags.Parse(args)
+	if err == flag.ErrHelp {
+		err = nil
+		return 0
+	} else if err != nil {
+		return 2
+	}
+
+	if len(flags.Args()) == 0 {
+		flags.Usage()
+		return 2
+	}
+
+	lvl, err := logrus.ParseLevel(*loglevel)
+	if err != nil {
+		return 2
+	}
+	logger.SetLevel(lvl)
+
+	cfg, err := loader.Load()
+	if err != nil {
+		return 1
+	}
+	cluster, err := cfg.GetCluster("")
+	if err != nil {
+		return 1
+	}
+	client, err := arvados.NewClientFromConfig(cluster)
+	if err != nil {
+		return 1
+	}
+	client.AuthToken = cluster.SystemRootToken
+	rcvr := recoverer{
+		client:  client,
+		cluster: cluster,
+		logger:  logger,
+	}
+
+	exitcode := 0
+	for _, src := range flags.Args() {
+		logger := logger.WithField("src", src)
+		var mtxt string
+		if !strings.Contains(src, "/") && len(src) == 27 && src[5] == '-' && src[11] == '-' {
+			var filters []arvados.Filter
+			if src[5:12] == "-57u5n-" {
+				filters = []arvados.Filter{{"uuid", "=", src}}
+			} else if src[5:12] == "-4zz18-" {
+				filters = []arvados.Filter{{"object_uuid", "=", src}}
+			} else {
+				logger.Error("looks like a UUID but not a log or collection UUID (if it's really a file, prepend './')")
+				exitcode = 1
+				continue
+			}
+			var resp struct {
+				Items []struct {
+					UUID       string    `json:"uuid"`
+					EventType  string    `json:"event_type"`
+					EventAt    time.Time `json:"event_at"`
+					ObjectUUID string    `json:"object_uuid"`
+					Properties struct {
+						OldAttributes struct {
+							ManifestText string `json:"manifest_text"`
+						} `json:"old_attributes"`
+					} `json:"properties"`
+				}
+			}
+			err = client.RequestAndDecode(&resp, "GET", "arvados/v1/logs", nil, arvados.ListOptions{
+				Limit:   1,
+				Order:   []string{"event_at desc"},
+				Filters: filters,
+			})
+			if err != nil {
+				logger.WithError(err).Error("error looking up log entry")
+				exitcode = 1
+				continue
+			} else if len(resp.Items) == 0 {
+				logger.Error("log entry not found")
+				exitcode = 1
+				continue
+			}
+			logent := resp.Items[0]
+			logger.WithFields(logrus.Fields{
+				"uuid":                logent.UUID,
+				"old_collection_uuid": logent.ObjectUUID,
+				"logged_event_type":   logent.EventType,
+				"logged_event_time":   logent.EventAt,
+				"logged_object_uuid":  logent.ObjectUUID,
+			}).Info("loaded log entry")
+			mtxt = logent.Properties.OldAttributes.ManifestText
+			if mtxt == "" {
+				logger.Error("log entry properties.old_attributes.manifest_text missing or empty")
+				exitcode = 1
+				continue
+			}
+		} else {
+			buf, err := ioutil.ReadFile(src)
+			if err != nil {
+				logger.WithError(err).Error("failed to load manifest data from file")
+				exitcode = 1
+				continue
+			}
+			mtxt = string(buf)
+		}
+		uuid, err := rcvr.RecoverManifest(string(mtxt))
+		if err != nil {
+			logger.WithError(err).Error("recovery failed")
+			exitcode = 1
+			continue
+		}
+		logger.WithField("UUID", uuid).Info("recovery succeeded")
+		fmt.Fprintln(stdout, uuid)
+	}
+	return exitcode
+}
+
+type recoverer struct {
+	client  *arvados.Client
+	cluster *arvados.Cluster
+	logger  logrus.FieldLogger
+}
+
+var errNotFound = errors.New("not found")
+
+// Finds the timestamp of the newest copy of blk on svc. Returns
+// errNotFound if blk is not on svc at all.
+func (rcvr recoverer) newestMtime(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
+	found, err := svc.Index(ctx, rcvr.client, blk)
+	if err != nil {
+		logger.WithError(err).Warn("error getting index")
+		return time.Time{}, err
+	} else if len(found) == 0 {
+		return time.Time{}, errNotFound
+	}
+	var latest time.Time
+	for _, ent := range found {
+		t := time.Unix(0, ent.Mtime)
+		if t.After(latest) {
+			latest = t
+		}
+	}
+	logger.WithField("latest", latest).Debug("found")
+	return latest, nil
+}
+
+var errTouchIneffective = errors.New("(BUG?) touch succeeded but had no effect -- reported timestamp is still too old")
+
+// Ensures the given block exists on the given server and won't be
+// eligible for trashing until after our chosen deadline (blobsigexp).
+// Returns an error if the block doesn't exist on the given server, or
+// has an old timestamp and can't be updated.
+//
+// After we decide a block is "safe" (whether or not we had to untrash
+// it), keep-balance might notice that it's currently unreferenced and
+// decide to trash it, all before our recovered collection gets
+// saved. But if the block's timestamp is more recent than blobsigttl,
+// keepstore will refuse to trash it even if told to by keep-balance.
+func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
+	if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err != nil {
+		return err
+	} else if latest.Add(blobsigttl).After(blobsigexp) {
+		return nil
+	}
+	if err := svc.Touch(ctx, rcvr.client, blk); err != nil {
+		return fmt.Errorf("error updating timestamp: %s", err)
+	}
+	logger.Debug("updated timestamp")
+	if latest, err := rcvr.newestMtime(ctx, logger, blk, svc); err == errNotFound {
+		return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
+	} else if err != nil {
+		return err
+	} else if latest.Add(blobsigttl).After(blobsigexp) {
+		return nil
+	} else {
+		return errTouchIneffective
+	}
+}
+
+// Untrash and update GC timestamps (as needed) on blocks referenced
+// by the given manifest, save a new collection and return the new
+// collection's UUID.
+func (rcvr recoverer) RecoverManifest(mtxt string) (string, error) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	coll := arvados.Collection{ManifestText: mtxt}
+	blks, err := coll.SizedDigests()
+	if err != nil {
+		return "", err
+	}
+	todo := make(chan int, len(blks))
+	for idx := range blks {
+		todo <- idx
+	}
+	go close(todo)
+
+	var services []arvados.KeepService
+	err = rcvr.client.EachKeepService(func(svc arvados.KeepService) error {
+		if svc.ServiceType == "proxy" {
+			rcvr.logger.WithField("service", svc).Debug("ignore proxy service")
+		} else {
+			services = append(services, svc)
+		}
+		return nil
+	})
+	if err != nil {
+		return "", fmt.Errorf("error getting list of keep services: %s", err)
+	}
+	rcvr.logger.WithField("services", services).Debug("got list of services")
+
+	// blobsigexp is our deadline for saving the rescued
+	// collection. This must be less than BlobSigningTTL
+	// (otherwise our rescued blocks could be garbage collected
+	// again before we protect them by saving the collection) but
+	// the exact value is somewhat arbitrary. If it's too soon, it
+	// will arrive before we're ready to save, and save will
+	// fail. If it's too late, we'll needlessly update timestamps
+	// on some blocks that were recently written/touched (e.g., by
+	// a previous attempt to rescue this same collection) and
+	// would have lived long enough anyway if left alone.
+	// BlobSigningTTL/2 (typically around 1 week) is much longer
+	// than than we need to recover even a very large collection.
+	blobsigttl := rcvr.cluster.Collections.BlobSigningTTL.Duration()
+	blobsigexp := time.Now().Add(blobsigttl / 2)
+	rcvr.logger.WithField("blobsigexp", blobsigexp).Debug("chose save deadline")
+
+	// We'll start a number of threads, each working on
+	// checking/recovering one block at a time. The threads
+	// themselves don't need much CPU/memory, but to avoid hitting
+	// limits on keepstore connections, backend storage bandwidth,
+	// etc., we limit concurrency to 2 per keepstore node.
+	workerThreads := 2 * len(services)
+
+	blkFound := make([]bool, len(blks))
+	var wg sync.WaitGroup
+	for i := 0; i < workerThreads; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+		nextblk:
+			for idx := range todo {
+				blk := strings.SplitN(string(blks[idx]), "+", 2)[0]
+				logger := rcvr.logger.WithField("block", blk)
+				for _, untrashing := range []bool{false, true} {
+					for _, svc := range services {
+						logger := logger.WithField("service", fmt.Sprintf("%s:%d", svc.ServiceHost, svc.ServicePort))
+						if untrashing {
+							if err := svc.Untrash(ctx, rcvr.client, blk); err != nil {
+								logger.WithError(err).Debug("untrash failed")
+								continue
+							}
+							logger.Info("untrashed")
+						}
+						err := rcvr.ensureSafe(ctx, logger, blk, svc, blobsigttl, blobsigexp)
+						if err == errNotFound {
+							logger.Debug(err)
+						} else if err != nil {
+							logger.Error(err)
+						} else {
+							blkFound[idx] = true
+							continue nextblk
+						}
+					}
+				}
+				logger.Debug("unrecoverable")
+			}
+		}()
+	}
+	wg.Wait()
+
+	var have, havenot int
+	for _, ok := range blkFound {
+		if ok {
+			have++
+		} else {
+			havenot++
+		}
+	}
+	if havenot > 0 {
+		if have > 0 {
+			rcvr.logger.Warn("partial recovery is not implemented")
+		}
+		return "", fmt.Errorf("unable to recover %d of %d blocks", havenot, have+havenot)
+	}
+
+	if rcvr.cluster.Collections.BlobSigning {
+		key := []byte(rcvr.cluster.Collections.BlobSigningKey)
+		coll.ManifestText = arvados.SignManifest(coll.ManifestText, rcvr.client.AuthToken, blobsigexp, blobsigttl, key)
+	}
+	rcvr.logger.WithField("manifest", coll.ManifestText).Debug("updated blob signatures in manifest")
+	err = rcvr.client.RequestAndDecodeContext(ctx, &coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+		"collection": map[string]interface{}{
+			"manifest_text": coll.ManifestText,
+		},
+	})
+	if err != nil {
+		return "", fmt.Errorf("error saving new collection: %s", err)
+	}
+	rcvr.logger.WithField("UUID", coll.UUID).Debug("created new collection")
+	return coll.UUID, nil
+}
diff --git a/lib/recovercollection/cmd_test.go b/lib/recovercollection/cmd_test.go
new file mode 100644
index 0000000000..57c2c64cda
--- /dev/null
+++ b/lib/recovercollection/cmd_test.go
@@ -0,0 +1,136 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package recovercollection
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+
+	"git.arvados.org/arvados.git/lib/config"
+	"git.arvados.org/arvados.git/sdk/go/arvadostest"
+	"git.arvados.org/arvados.git/sdk/go/ctxlog"
+	"gopkg.in/check.v1"
+)
+
+func Test(t *testing.T) {
+	check.TestingT(t)
+}
+
+var _ = check.Suite(&Suite{})
+
+type Suite struct{}
+
+func (*Suite) SetUpSuite(c *check.C) {
+	arvadostest.StartAPI()
+	arvadostest.StartKeep(2, true)
+}
+
+func (*Suite) TestUnrecoverableBlock(c *check.C) {
+	tmp := c.MkDir()
+	mfile := tmp + "/manifest"
+	ioutil.WriteFile(mfile, []byte(". aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+410 0:410:Gone\n"), 0777)
+	var stdout, stderr bytes.Buffer
+	exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 1)
+	c.Check(stdout.String(), check.Equals, "")
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*msg="not found" block=aaaaa.*`)
+	c.Check(stderr.String(), check.Matches, `(?ms).*msg="untrash failed" block=aaaaa.*`)
+	c.Check(stderr.String(), check.Matches, `(?ms).*msg=unrecoverable block=aaaaa.*`)
+	c.Check(stderr.String(), check.Matches, `(?ms).*msg="recovery failed".*`)
+}
+
+func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
+	tmp := c.MkDir()
+	mfile := tmp + "/manifest"
+	ioutil.WriteFile(mfile, []byte(". dcd0348cb2532ee90c99f1b846efaee7+13 0:13:test.txt\n"), 0777)
+
+	logger := ctxlog.TestLogger(c)
+	loader := config.NewLoader(&bytes.Buffer{}, logger)
+	cfg, err := loader.Load()
+	c.Assert(err, check.IsNil)
+	cluster, err := cfg.GetCluster("")
+	c.Assert(err, check.IsNil)
+	var datadirs []string
+	for _, v := range cluster.Volumes {
+		var params struct {
+			Root string
+		}
+		err := json.Unmarshal(v.DriverParameters, &params)
+		c.Assert(err, check.IsNil)
+		if params.Root != "" {
+			datadirs = append(datadirs, params.Root)
+			err := os.Remove(params.Root + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+			if err != nil && !os.IsNotExist(err) {
+				c.Error(err)
+			}
+		}
+	}
+	c.Logf("keepstore datadirs are %q", datadirs)
+
+	// Currently StartKeep(2, true) uses dirs called "keep0" and
+	// "keep1" so we could just put our fake trashed file in keep0
+	// ... but we don't want to rely on arvadostest's
+	// implementation details, so we put a trashed file in every
+	// dir that keepstore might be using.
+	for _, datadir := range datadirs {
+		if fi, err := os.Stat(datadir); err != nil || !fi.IsDir() {
+			continue
+		}
+		c.Logf("placing backdated trashed block in datadir %q", datadir)
+		trashfile := datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7.trash.999999999"
+		os.Mkdir(datadir+"/dcd", 0777)
+		err = ioutil.WriteFile(trashfile, []byte("undelete test"), 0777)
+		c.Assert(err, check.IsNil)
+		t := time.Now().Add(-time.Hour * 24 * 365)
+		err = os.Chtimes(trashfile, t, t)
+	}
+
+	var stdout, stderr bytes.Buffer
+	exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+	c.Check(exitcode, check.Equals, 0)
+	c.Check(stdout.String(), check.Matches, `zzzzz-4zz18-.{15}\n`)
+	c.Log(stderr.String())
+	c.Check(stderr.String(), check.Matches, `(?ms).*msg=untrashed block=dcd0348.*`)
+	c.Check(stderr.String(), check.Matches, `(?ms).*msg="updated timestamp" block=dcd0348.*`)
+
+	found := false
+	for _, datadir := range datadirs {
+		buf, err := ioutil.ReadFile(datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+		if err == nil {
+			found = true
+			c.Check(buf, check.DeepEquals, []byte("undelete test"))
+			fi, err := os.Stat(datadir + "/dcd/dcd0348cb2532ee90c99f1b846efaee7")
+			if c.Check(err, check.IsNil) {
+				c.Logf("recovered block's modtime is %s", fi.ModTime())
+				c.Check(time.Now().Sub(fi.ModTime()) < time.Hour, check.Equals, true)
+			}
+		}
+	}
+	c.Check(found, check.Equals, true)
+}
+
+func (*Suite) TestUnusableManifestSourceArg(c *check.C) {
+	for _, trial := range []struct {
+		srcArg    string
+		errRegexp string
+	}{
+		{"zzzzz-4zz18-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found".*`},
+		{"zzzzz-57u5n-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found.*`},
+		{"zzzzz-57u5n-containerlog006", `(?ms).*msg="log entry properties\.old_attributes\.manifest_text missing or empty".*`},
+		{"zzzzz-j7d0g-aaaaaaaaaaaaaaa", `(?ms).*msg="looks like a UUID but not a log or collection UUID.*`},
+	} {
+		var stdout, stderr bytes.Buffer
+		exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", trial.srcArg}, &bytes.Buffer{}, &stdout, &stderr)
+		c.Check(exitcode, check.Equals, 1)
+		c.Check(stdout.String(), check.Equals, "")
+		c.Log(stderr.String())
+		c.Check(stderr.String(), check.Matches, trial.errRegexp)
+	}
+}
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index 1e7a9a36ed..901fda2289 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -177,6 +177,9 @@ func getListenAddr(svcs arvados.Services, prog arvados.ServiceName, log logrus.F
 	} else if url, err := url.Parse(want); err != nil {
 		return arvados.URL{}, fmt.Errorf("$ARVADOS_SERVICE_INTERNAL_URL (%q): %s", want, err)
 	} else {
+		if url.Path == "" {
+			url.Path = "/"
+		}
 		return arvados.URL(*url), nil
 	}
 
diff --git a/sdk/R/install_deps.R b/sdk/R/install_deps.R
index 593129bb3c..6c33f97913 100644
--- a/sdk/R/install_deps.R
+++ b/sdk/R/install_deps.R
@@ -15,5 +15,11 @@ if (!requireNamespace("knitr")) {
 if (!requireNamespace("markdown")) {
   install.packages("markdown")
 }
+if (!requireNamespace("XML")) {
+  # XML 3.99-0.4 depends on R >= 4.0.0, but we run tests on debian
+  # stable (10) with R 3.5.2 so we install an older version from
+  # source.
+  install.packages("https://cran.r-project.org/src/contrib/Archive/XML/XML_3.99-0.3.tar.gz", repos=NULL, type="source")
+}
 
 devtools::install_dev_deps()
diff --git a/sdk/cwl/arvados_cwl/__init__.py b/sdk/cwl/arvados_cwl/__init__.py
index 5dde027732..6f2255b3f8 100644
--- a/sdk/cwl/arvados_cwl/__init__.py
+++ b/sdk/cwl/arvados_cwl/__init__.py
@@ -23,7 +23,7 @@ import cwltool.workflow
 import cwltool.process
 import cwltool.argparser
 from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing
 
 import arvados
 import arvados.config
@@ -220,8 +220,8 @@ def add_arv_hints():
     cwltool.command_line_tool.ACCEPTLIST_RE = cwltool.command_line_tool.ACCEPTLIST_EN_RELAXED_RE
     res10 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.0.yml')
     res11 = pkg_resources.resource_stream(__name__, 'arv-cwl-schema-v1.1.yml')
-    customschema10 = res10.read()
-    customschema11 = res11.read()
+    customschema10 = res10.read().decode('utf-8')
+    customschema11 = res11.read().decode('utf-8')
     use_custom_schema("v1.0", "http://arvados.org/cwl", customschema10)
     use_custom_schema("v1.1.0-dev1", "http://arvados.org/cwl", customschema11)
     use_custom_schema("v1.1", "http://arvados.org/cwl", customschema11)
diff --git a/sdk/cwl/arvados_cwl/executor.py b/sdk/cwl/arvados_cwl/executor.py
index 99d4c4e9a1..ec91eea6aa 100644
--- a/sdk/cwl/arvados_cwl/executor.py
+++ b/sdk/cwl/arvados_cwl/executor.py
@@ -42,7 +42,7 @@ from .context import ArvLoadingContext, ArvRuntimeContext
 from ._version import __version__
 
 from cwltool.process import shortname, UnsupportedRequirement, use_custom_schema
-from cwltool.pathmapper import adjustFileObjs, adjustDirObjs, get_listing, visit_class
+from cwltool.utils import adjustFileObjs, adjustDirObjs, get_listing, visit_class
 from cwltool.command_line_tool import compute_checksums
 from cwltool.load_tool import load_tool
 
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index f0be830324..71e499ebca 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -42,6 +42,7 @@ import schema_salad.validate as validate
 import arvados.collection
 from .util import collectionUUID
 import ruamel.yaml as yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 import arvados_cwl.arvdocker
 from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern
@@ -168,21 +169,47 @@ def set_secondary(fsaccess, builder, inputschema, secondaryspec, primary, discov
         #
         # Found a file, check for secondaryFiles
         #
-        primary["secondaryFiles"] = []
+        specs = []
+        primary["secondaryFiles"] = secondaryspec
         for i, sf in enumerate(aslist(secondaryspec)):
             pattern = builder.do_eval(sf["pattern"], context=primary)
             if pattern is None:
                 continue
+            if isinstance(pattern, list):
+                specs.extend(pattern)
+            elif isinstance(pattern, dict):
+                specs.append(pattern)
+            elif isinstance(pattern, str):
+                specs.append({"pattern": pattern})
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
+        found = []
+        for i, sf in enumerate(specs):
+            if isinstance(sf, dict):
+                if sf.get("class") == "File":
+                    pattern = sf["basename"]
+                else:
+                    pattern = sf["pattern"]
+                    required = sf.get("required")
+            elif isinstance(sf, str):
+                pattern = sf
+                required = True
+            else:
+                raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
+                    "Expression must return list, object, string or null")
+
             sfpath = substitute(primary["location"], pattern)
-            required = builder.do_eval(sf.get("required"), context=primary)
+            required = builder.do_eval(required, context=primary)
 
             if fsaccess.exists(sfpath):
-                primary["secondaryFiles"].append({"location": sfpath, "class": "File"})
+                found.append({"location": sfpath, "class": "File"})
             elif required:
                 raise SourceLine(primary["secondaryFiles"], i, validate.ValidationException).makeError(
                     "Required secondary file '%s' does not exist" % sfpath)
 
-        primary["secondaryFiles"] = cmap(primary["secondaryFiles"])
+        primary["secondaryFiles"] = cmap(found)
         if discovered is not None:
             discovered[primary["location"]] = primary["secondaryFiles"]
     elif inputschema["type"] not in primitive_types_set:
@@ -392,7 +419,7 @@ def upload_dependencies(arvrunner, name, document_loader,
             discovered_secondaryfiles[mapper.mapper(d).resolved] = discovered[d]
 
     if "$schemas" in workflowobj:
-        sch = []
+        sch = CommentedSeq()
         for s in workflowobj["$schemas"]:
             sch.append(mapper.mapper(s).resolved)
         workflowobj["$schemas"] = sch
@@ -433,9 +460,13 @@ def packed_workflow(arvrunner, tool, merged_map):
     def visit(v, cur_id):
         if isinstance(v, dict):
             if v.get("class") in ("CommandLineTool", "Workflow"):
-                if "id" not in v:
-                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field")
-                cur_id = rewrite_to_orig.get(v["id"], v["id"])
+                if tool.metadata["cwlVersion"] == "v1.0" and "id" not in v:
+                    raise SourceLine(v, None, Exception).makeError("Embedded process object is missing required 'id' field, add an 'id' or use to cwlVersion: v1.1")
+                if "id" in v:
+                    cur_id = rewrite_to_orig.get(v["id"], v["id"])
+            if "path" in v and "location" not in v:
+                v["location"] = v["path"]
+                del v["path"]
             if "location" in v and not v["location"].startswith("keep:"):
                 v["location"] = merged_map[cur_id].resolved[v["location"]]
             if "location" in v and v["location"] in merged_map[cur_id].secondaryFiles:
diff --git a/sdk/cwl/fpm-info.sh b/sdk/cwl/fpm-info.sh
index 5c47532db9..66176b940b 100644
--- a/sdk/cwl/fpm-info.sh
+++ b/sdk/cwl/fpm-info.sh
@@ -6,8 +6,11 @@ case "$TARGET" in
     debian8)
         fpm_depends+=(libgnutls-deb0-28 libcurl3-gnutls)
         ;;
+    debian9 | ubuntu1604)
+        fpm_depends+=(libcurl3-gnutls)
+        ;;
     debian* | ubuntu*)
-        fpm_depends+=(libcurl3-gnutls libpython2.7)
+        fpm_depends+=(libcurl3-gnutls python3-distutils)
         ;;
 esac
 
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 95730a69b1..40ee679857 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -39,8 +39,8 @@ setup(name='arvados-cwl-runner',
       # file to determine what version of cwltool and schema-salad to
       # build.
       install_requires=[
-          'cwltool==3.0.20200317203547',
-          'schema-salad==5.0.20200302192450',
+          'cwltool==3.0.20200530110633',
+          'schema-salad==6.0.20200601095207',
           'arvados-python-client{}'.format(pysdk_dep),
           'setuptools',
           'ciso8601 >= 2.0.0'
diff --git a/sdk/cwl/test_with_arvbox.sh b/sdk/cwl/test_with_arvbox.sh
index 76aa43d611..6de404f448 100755
--- a/sdk/cwl/test_with_arvbox.sh
+++ b/sdk/cwl/test_with_arvbox.sh
@@ -98,7 +98,7 @@ fi
 
 set -x
 
-if [ \$PYCMD = "python3" ]; then
+if [ "\$PYCMD" = "python3" ]; then
     pip3 install cwltest
 else
     pip install cwltest
@@ -118,6 +118,9 @@ elif [[ "$suite" =~ conformance-(.*) ]] ; then
      git clone https://github.com/common-workflow-language/cwl-\${version}.git
    fi
    cd cwl-\${version}
+elif [[ "$suite" != "integration" ]] ; then
+   echo "ERROR: unknown suite '$suite'"
+   exit 1
 fi
 
 if [[ "$suite" != "integration" ]] ; then
@@ -133,9 +136,17 @@ if test -n "$build" ; then
 elif test "$tag" = "latest" ; then
   arv-keepdocker --pull arvados/jobs $tag
 else
-  jobsimg=\$(curl https://versions.arvados.org/v1/commit/$tag | python -c "import json; import sys; sys.stdout.write(json.load(sys.stdin)['Versions']['Docker']['arvados/jobs'])")
-  arv-keepdocker --pull arvados/jobs \$jobsimg
-  docker tag arvados/jobs:\$jobsimg arvados/jobs:latest
+  set +u
+  export WORKSPACE=/usr/src/arvados
+  . /usr/src/arvados/build/run-library.sh
+  TMPHERE=\$(pwd)
+  cd /usr/src/arvados
+  calculate_python_sdk_cwl_package_versions
+  cd \$TMPHERE
+  set -u
+
+  arv-keepdocker --pull arvados/jobs \$cwl_runner_version
+  docker tag arvados/jobs:\$cwl_runner_version arvados/jobs:latest
   arv-keepdocker arvados/jobs latest
 fi
 
@@ -153,7 +164,7 @@ chmod +x /tmp/cwltest/arv-cwl-containers
 
 EXTRA=--compute-checksum
 
-if [[ $devcwl == 1 ]] ; then
+if [[ $devcwl -eq 1 ]] ; then
    EXTRA="\$EXTRA --enable-dev"
 fi
 
diff --git a/sdk/cwl/tests/arvados-tests.yml b/sdk/cwl/tests/arvados-tests.yml
index df9fac8426..c4c0968756 100644
--- a/sdk/cwl/tests/arvados-tests.yml
+++ b/sdk/cwl/tests/arvados-tests.yml
@@ -218,9 +218,16 @@
   output:
     out: null
   tool: wf-defaults/wf4.cwl
-  doc: default in embedded subworkflow missing 'id' field
+  doc: default in embedded subworkflow missing 'id' field, v1.0
   should_fail: true
 
+- job: null
+  output:
+    out: null
+  tool: wf-defaults/wf8.cwl
+  doc: default in embedded subworkflow missing 'id' field, v1.1
+  should_fail: false
+
 - job: null
   output:
     out: null
diff --git a/sdk/cwl/tests/federation/arvbox-make-federation.cwl b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
index 593f2399f5..ed3f1d1384 100644
--- a/sdk/cwl/tests/federation/arvbox-make-federation.cwl
+++ b/sdk/cwl/tests/federation/arvbox-make-federation.cwl
@@ -33,6 +33,9 @@ inputs:
   logincluster:
     type: boolean
     default: false
+  arvbox_mode:
+    type: string?
+    default: "dev"
 outputs:
   arvados_api_token:
     type: string
@@ -71,6 +74,7 @@ steps:
       arvbox_data: mkdir/arvbox_data
       arvbox_bin: arvbox
       branch: branch
+      arvbox_mode: arvbox_mode
     out: [cluster_id, container_host, arvbox_data_out, superuser_token]
     scatter: [container_name, arvbox_data]
     scatterMethod: dotproduct
diff --git a/sdk/cwl/tests/federation/arvboxcwl/start.cwl b/sdk/cwl/tests/federation/arvboxcwl/start.cwl
index a7f46d6b22..c933de254a 100644
--- a/sdk/cwl/tests/federation/arvboxcwl/start.cwl
+++ b/sdk/cwl/tests/federation/arvboxcwl/start.cwl
@@ -14,6 +14,9 @@ inputs:
   branch:
     type: string
     default: master
+  arvbox_mode:
+    type: string?
+    default: "dev"
 outputs:
   cluster_id:
     type: string
@@ -71,24 +74,28 @@ arguments:
   - shellQuote: false
     valueFrom: |
       set -ex
-      mkdir -p $ARVBOX_DATA
-      if ! test -d $ARVBOX_DATA/arvados ; then
-        cd $ARVBOX_DATA
-        git clone https://git.arvados.org/arvados.git
+      if test $(inputs.arvbox_mode) = dev ; then
+        mkdir -p $ARVBOX_DATA
+        if ! test -d $ARVBOX_DATA/arvados ; then
+          cd $ARVBOX_DATA
+          git clone https://git.arvados.org/arvados.git
+        fi
+        cd $ARVBOX_DATA/arvados
+        gitver=`git rev-parse HEAD`
+        git fetch
+        git checkout -f $(inputs.branch)
+        git pull
+        pulled=`git rev-parse HEAD`
+        git --no-pager log -n1 $pulled
+      else
+        export ARVBOX_BASE=$(runtime.tmpdir)
+        unset ARVBOX_DATA
       fi
-      cd $ARVBOX_DATA/arvados
-      gitver=`git rev-parse HEAD`
-      git fetch
-      git checkout -f $(inputs.branch)
-      git pull
-      pulled=`git rev-parse HEAD`
-      git --no-pager log -n1 $pulled
-
       cd $(runtime.outdir)
       if test "$gitver" = "$pulled" ; then
-        $(inputs.arvbox_bin.path) start dev
+        $(inputs.arvbox_bin.path) start $(inputs.arvbox_mode)
       else
-        $(inputs.arvbox_bin.path) restart dev
+        $(inputs.arvbox_bin.path) restart $(inputs.arvbox_mode)
       fi
       $(inputs.arvbox_bin.path) status > status.txt
       $(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
diff --git a/sdk/cwl/tests/wf-defaults/default-dir4.cwl b/sdk/cwl/tests/wf-defaults/default-dir4.cwl
index 8bfc5d63f7..bd92782488 100644
--- a/sdk/cwl/tests/wf-defaults/default-dir4.cwl
+++ b/sdk/cwl/tests/wf-defaults/default-dir4.cwl
@@ -21,4 +21,4 @@ steps:
             class: Directory
             location: inp1
       outputs: []
-      arguments: [echo, $(inputs.inp2)]
\ No newline at end of file
+      arguments: [echo, $(inputs.inp2)]
diff --git a/sdk/cwl/tests/wf-defaults/default-dir8.cwl b/sdk/cwl/tests/wf-defaults/default-dir8.cwl
new file mode 100644
index 0000000000..a5b9c2f3ec
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/default-dir8.cwl
@@ -0,0 +1,24 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+steps:
+  step1:
+    in: []
+    out: []
+    run:
+      class: CommandLineTool
+      inputs:
+        inp2:
+          type: Directory
+          default:
+            class: Directory
+            location: inp1
+      outputs: []
+      arguments: [echo, $(inputs.inp2)]
diff --git a/sdk/cwl/tests/wf-defaults/wf4.cwl b/sdk/cwl/tests/wf-defaults/wf4.cwl
index 6e562e43db..3f498fdffb 100644
--- a/sdk/cwl/tests/wf-defaults/wf4.cwl
+++ b/sdk/cwl/tests/wf-defaults/wf4.cwl
@@ -14,4 +14,4 @@ steps:
   step1:
     in: []
     out: []
-    run: default-dir4.cwl
\ No newline at end of file
+    run: default-dir4.cwl
diff --git a/sdk/cwl/tests/wf-defaults/wf8.cwl b/sdk/cwl/tests/wf-defaults/wf8.cwl
new file mode 100644
index 0000000000..2548faec1c
--- /dev/null
+++ b/sdk/cwl/tests/wf-defaults/wf8.cwl
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+cwlVersion: v1.1
+class: Workflow
+inputs: []
+outputs: []
+$namespaces:
+  arv: "http://arvados.org/cwl#"
+requirements:
+  SubworkflowFeatureRequirement: {}
+steps:
+  step1:
+    in: []
+    out: []
+    run: default-dir8.cwl
diff --git a/sdk/go/arvados/blob_signature.go b/sdk/go/arvados/blob_signature.go
new file mode 100644
index 0000000000..132939547a
--- /dev/null
+++ b/sdk/go/arvados/blob_signature.go
@@ -0,0 +1,126 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+// Generate and verify permission signatures for Keep locators.
+//
+// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
+
+package arvados
+
+import (
+	"crypto/hmac"
+	"crypto/sha1"
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+var (
+	// ErrSignatureExpired - a signature was rejected because the
+	// expiry time has passed.
+	ErrSignatureExpired = errors.New("Signature expired")
+	// ErrSignatureInvalid - a signature was rejected because it
+	// was badly formatted or did not match the given secret key.
+	ErrSignatureInvalid = errors.New("Invalid signature")
+	// ErrSignatureMissing - the given locator does not have a
+	// signature hint.
+	ErrSignatureMissing = errors.New("Missing signature")
+)
+
+// makePermSignature generates a SHA-1 HMAC digest for the given blob,
+// token, expiry, and site secret.
+func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
+	hmac := hmac.New(sha1.New, permissionSecret)
+	hmac.Write([]byte(blobHash))
+	hmac.Write([]byte("@"))
+	hmac.Write([]byte(apiToken))
+	hmac.Write([]byte("@"))
+	hmac.Write([]byte(expiry))
+	hmac.Write([]byte("@"))
+	hmac.Write([]byte(blobSignatureTTL))
+	digest := hmac.Sum(nil)
+	return fmt.Sprintf("%x", digest)
+}
+
+var (
+	mBlkRe      = regexp.MustCompile(`^[0-9a-f]{32}.*`)
+	mPermHintRe = regexp.MustCompile(`\+A[^+]*`)
+)
+
+// SignManifest signs all locators in the given manifest, discarding
+// any existing signatures.
+func SignManifest(manifest string, apiToken string, expiry time.Time, ttl time.Duration, permissionSecret []byte) string {
+	return regexp.MustCompile(`\S+`).ReplaceAllStringFunc(manifest, func(tok string) string {
+		if mBlkRe.MatchString(tok) {
+			return SignLocator(mPermHintRe.ReplaceAllString(tok, ""), apiToken, expiry, ttl, permissionSecret)
+		} else {
+			return tok
+		}
+	})
+}
+
+// SignLocator returns blobLocator with a permission signature
+// added. If either permissionSecret or apiToken is empty, blobLocator
+// is returned untouched.
+//
+// This function is intended to be used by system components and admin
+// utilities: userland programs do not know the permissionSecret.
+func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
+	if len(permissionSecret) == 0 || apiToken == "" {
+		return blobLocator
+	}
+	// Strip off all hints: only the hash is used to sign.
+	blobHash := strings.Split(blobLocator, "+")[0]
+	timestampHex := fmt.Sprintf("%08x", expiry.Unix())
+	blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
+	return blobLocator +
+		"+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
+		"@" + timestampHex
+}
+
+var SignedLocatorRe = regexp.MustCompile(
+	//1                 2          34                         5   6                  7                 89
+	`^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9@_-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9@_-]*)*)$`)
+
+// VerifySignature returns nil if the signature on the signedLocator
+// can be verified using the given apiToken. Otherwise it returns
+// ErrSignatureExpired (if the signature's expiry time has passed,
+// which is something the client could have figured out
+// independently), ErrSignatureMissing (if there is no signature hint
+// at all), or ErrSignatureInvalid (if the signature is present but
+// badly formatted or incorrect).
+//
+// This function is intended to be used by system components and admin
+// utilities: userland programs do not know the permissionSecret.
+func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
+	matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
+	if matches == nil {
+		return ErrSignatureMissing
+	}
+	blobHash := matches[1]
+	signatureHex := matches[6]
+	expiryHex := matches[7]
+	if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
+		return ErrSignatureInvalid
+	} else if expiryTime.Before(time.Now()) {
+		return ErrSignatureExpired
+	}
+	blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
+	if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
+		return ErrSignatureInvalid
+	}
+	return nil
+}
+
+func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
+	if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
+		ts = time.Unix(tsInt, 0)
+	} else {
+		err = e
+	}
+	return ts, err
+}
diff --git a/sdk/go/arvados/blob_signature_test.go b/sdk/go/arvados/blob_signature_test.go
new file mode 100644
index 0000000000..847f9a8ae2
--- /dev/null
+++ b/sdk/go/arvados/blob_signature_test.go
@@ -0,0 +1,88 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import (
+	"time"
+
+	check "gopkg.in/check.v1"
+)
+
+const (
+	knownHash    = "acbd18db4cc2f85cedef654fccc4a4d8"
+	knownLocator = knownHash + "+3"
+	knownToken   = "hocfupkn2pjhrpgp2vxv8rsku7tvtx49arbc9s4bvu7p7wxqvk"
+	knownKey     = "13u9fkuccnboeewr0ne3mvapk28epf68a3bhj9q8sb4l6e4e5mkk" +
+		"p6nhj2mmpscgu1zze5h5enydxfe3j215024u16ij4hjaiqs5u4pzsl3nczmaoxnc" +
+		"ljkm4875xqn4xv058koz3vkptmzhyheiy6wzevzjmdvxhvcqsvr5abhl15c2d4o4" +
+		"jhl0s91lojy1mtrzqqvprqcverls0xvy9vai9t1l1lvvazpuadafm71jl4mrwq2y" +
+		"gokee3eamvjy8qq1fvy238838enjmy5wzy2md7yvsitp5vztft6j4q866efym7e6" +
+		"vu5wm9fpnwjyxfldw3vbo01mgjs75rgo7qioh8z8ij7jpyp8508okhgbbex3ceei" +
+		"786u5rw2a9gx743dj3fgq2irk"
+	knownSignature     = "89118b78732c33104a4d6231e8b5a5fa1e4301e3"
+	knownTimestamp     = "7fffffff"
+	knownSigHint       = "+A" + knownSignature + "@" + knownTimestamp
+	knownSignedLocator = knownLocator + knownSigHint
+	blobSignatureTTL   = 1209600 * time.Second
+)
+
+var _ = check.Suite(&BlobSignatureSuite{})
+
+type BlobSignatureSuite struct{}
+
+func (s *BlobSignatureSuite) TestSignLocator(c *check.C) {
+	ts, err := parseHexTimestamp(knownTimestamp)
+	c.Check(err, check.IsNil)
+	c.Check(SignLocator(knownLocator, knownToken, ts, blobSignatureTTL, []byte(knownKey)), check.Equals, knownSignedLocator)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignature(c *check.C) {
+	c.Check(VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureExtraHints(c *check.C) {
+	// handle hint before permission signature
+	c.Check(VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+	// handle hint after permission signature
+	c.Check(VerifySignature(knownLocator+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+	// handle hints around permission signature
+	c.Check(VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+// The size hint on the locator string should not affect signature
+// validation.
+func (s *BlobSignatureSuite) TestVerifySignatureWrongSize(c *check.C) {
+	// handle incorrect size hint
+	c.Check(VerifySignature(knownHash+"+999999"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+
+	// handle missing size hint
+	c.Check(VerifySignature(knownHash+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)), check.IsNil)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadSig(c *check.C) {
+	badLocator := knownLocator + "+Aaaaaaaaaaaaaaaa@" + knownTimestamp
+	c.Check(VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureMissing)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadTimestamp(c *check.C) {
+	badLocator := knownLocator + "+A" + knownSignature + "@OOOOOOOl"
+	c.Check(VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureMissing)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadSecret(c *check.C) {
+	c.Check(VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte("00000000000000000000")), check.Equals, ErrSignatureInvalid)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureBadToken(c *check.C) {
+	c.Check(VerifySignature(knownSignedLocator, "00000000", blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureInvalid)
+}
+
+func (s *BlobSignatureSuite) TestVerifySignatureExpired(c *check.C) {
+	yesterday := time.Now().AddDate(0, 0, -1)
+	expiredLocator := SignLocator(knownHash, knownToken, yesterday, blobSignatureTTL, []byte(knownKey))
+	c.Check(VerifySignature(expiredLocator, knownToken, blobSignatureTTL, []byte(knownKey)), check.Equals, ErrSignatureExpired)
+}
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 1e2c07e867..562c8c1e7d 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -57,9 +57,16 @@ type Client struct {
 	// HTTP headers to add/override in outgoing requests.
 	SendHeader http.Header
 
+	// Timeout for requests. NewClientFromConfig and
+	// NewClientFromEnv return a Client with a default 5 minute
+	// timeout.  To disable this timeout and rely on each
+	// http.Request's context deadline instead, set Timeout to
+	// zero.
+	Timeout time.Duration
+
 	dd *DiscoveryDocument
 
-	ctx context.Context
+	defaultRequestID string
 }
 
 // The default http.Client used by a Client with Insecure==true and
@@ -67,12 +74,10 @@ type Client struct {
 var InsecureHTTPClient = &http.Client{
 	Transport: &http.Transport{
 		TLSClientConfig: &tls.Config{
-			InsecureSkipVerify: true}},
-	Timeout: 5 * time.Minute}
+			InsecureSkipVerify: true}}}
 
 // The default http.Client used by a Client otherwise.
-var DefaultSecureClient = &http.Client{
-	Timeout: 5 * time.Minute}
+var DefaultSecureClient = &http.Client{}
 
 // NewClientFromConfig creates a new Client that uses the endpoints in
 // the given cluster.
@@ -87,6 +92,7 @@ func NewClientFromConfig(cluster *Cluster) (*Client, error) {
 		Scheme:   ctrlURL.Scheme,
 		APIHost:  ctrlURL.Host,
 		Insecure: cluster.TLS.Insecure,
+		Timeout:  5 * time.Minute,
 	}, nil
 }
 
@@ -116,6 +122,7 @@ func NewClientFromEnv() *Client {
 		AuthToken:       os.Getenv("ARVADOS_API_TOKEN"),
 		Insecure:        insecure,
 		KeepServiceURIs: svcs,
+		Timeout:         5 * time.Minute,
 	}
 }
 
@@ -131,11 +138,12 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 	}
 
 	if req.Header.Get("X-Request-Id") == "" {
-		reqid, _ := req.Context().Value(contextKeyRequestID{}).(string)
-		if reqid == "" {
-			reqid, _ = c.context().Value(contextKeyRequestID{}).(string)
-		}
-		if reqid == "" {
+		var reqid string
+		if ctxreqid, _ := req.Context().Value(contextKeyRequestID{}).(string); ctxreqid != "" {
+			reqid = ctxreqid
+		} else if c.defaultRequestID != "" {
+			reqid = c.defaultRequestID
+		} else {
 			reqid = reqIDGen.Next()
 		}
 		if req.Header == nil {
@@ -144,7 +152,36 @@ func (c *Client) Do(req *http.Request) (*http.Response, error) {
 			req.Header.Set("X-Request-Id", reqid)
 		}
 	}
-	return c.httpClient().Do(req)
+	var cancel context.CancelFunc
+	if c.Timeout > 0 {
+		ctx := req.Context()
+		ctx, cancel = context.WithDeadline(ctx, time.Now().Add(c.Timeout))
+		req = req.WithContext(ctx)
+	}
+	resp, err := c.httpClient().Do(req)
+	if err == nil && cancel != nil {
+		// We need to call cancel() eventually, but we can't
+		// use "defer cancel()" because the context has to
+		// stay alive until the caller has finished reading
+		// the response body.
+		resp.Body = cancelOnClose{ReadCloser: resp.Body, cancel: cancel}
+	} else if cancel != nil {
+		cancel()
+	}
+	return resp, err
+}
+
+// cancelOnClose calls a provided CancelFunc when its wrapped
+// ReadCloser's Close() method is called.
+type cancelOnClose struct {
+	io.ReadCloser
+	cancel context.CancelFunc
+}
+
+func (coc cancelOnClose) Close() error {
+	err := coc.ReadCloser.Close()
+	coc.cancel()
+	return err
 }
 
 func isRedirectStatus(code int) bool {
@@ -266,7 +303,7 @@ func anythingToValues(params interface{}) (url.Values, error) {
 //
 // path must not contain a query string.
 func (c *Client) RequestAndDecode(dst interface{}, method, path string, body io.Reader, params interface{}) error {
-	return c.RequestAndDecodeContext(c.context(), dst, method, path, body, params)
+	return c.RequestAndDecodeContext(context.Background(), dst, method, path, body, params)
 }
 
 func (c *Client) RequestAndDecodeContext(ctx context.Context, dst interface{}, method, path string, body io.Reader, params interface{}) error {
@@ -332,17 +369,10 @@ func (c *Client) UpdateBody(rsc resource) io.Reader {
 // header.
 func (c *Client) WithRequestID(reqid string) *Client {
 	cc := *c
-	cc.ctx = ContextWithRequestID(cc.context(), reqid)
+	cc.defaultRequestID = reqid
 	return &cc
 }
 
-func (c *Client) context() context.Context {
-	if c.ctx == nil {
-		return context.Background()
-	}
-	return c.ctx
-}
-
 func (c *Client) httpClient() *http.Client {
 	switch {
 	case c.Client != nil:
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 6b83fb96d4..a54712f330 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -23,7 +23,8 @@ var DefaultConfigFile = func() string {
 }()
 
 type Config struct {
-	Clusters map[string]Cluster
+	Clusters         map[string]Cluster
+	AutoReloadConfig bool
 }
 
 // GetConfig returns the current system config, loading it from
@@ -66,6 +67,7 @@ type WebDAVCacheConfig struct {
 	MaxPermissionEntries int
 	MaxUUIDEntries       int
 }
+
 type Cluster struct {
 	ClusterID       string `json:"-"`
 	ManagementToken string
@@ -124,6 +126,7 @@ type Cluster struct {
 		BalancePeriod            Duration
 		BalanceCollectionBatch   int
 		BalanceCollectionBuffers int
+		BalanceTimeout           Duration
 
 		WebDAVCache WebDAVCacheConfig
 	}
@@ -133,16 +136,48 @@ type Cluster struct {
 		Repositories string
 	}
 	Login struct {
-		GoogleClientID                string
-		GoogleClientSecret            string
-		GoogleAlternateEmailAddresses bool
-		PAM                           bool
-		PAMService                    string
-		PAMDefaultEmailDomain         string
-		ProviderAppID                 string
-		ProviderAppSecret             string
-		LoginCluster                  string
-		RemoteTokenRefresh            Duration
+		LDAP struct {
+			Enable             bool
+			URL                URL
+			StartTLS           bool
+			InsecureTLS        bool
+			StripDomain        string
+			AppendDomain       string
+			SearchAttribute    string
+			SearchBindUser     string
+			SearchBindPassword string
+			SearchBase         string
+			SearchFilters      string
+			EmailAttribute     string
+			UsernameAttribute  string
+		}
+		Google struct {
+			Enable                  bool
+			ClientID                string
+			ClientSecret            string
+			AlternateEmailAddresses bool
+		}
+		OpenIDConnect struct {
+			Enable             bool
+			Issuer             string
+			ClientID           string
+			ClientSecret       string
+			EmailClaim         string
+			EmailVerifiedClaim string
+			UsernameClaim      string
+		}
+		PAM struct {
+			Enable             bool
+			Service            string
+			DefaultEmailDomain string
+		}
+		SSO struct {
+			Enable            bool
+			ProviderAppID     string
+			ProviderAppSecret string
+		}
+		LoginCluster       string
+		RemoteTokenRefresh Duration
 	}
 	Mail struct {
 		MailchimpAPIKey                string
@@ -234,12 +269,14 @@ type Volume struct {
 }
 
 type S3VolumeDriverParameters struct {
+	IAMRole            string
 	AccessKey          string
 	SecretKey          string
 	Endpoint           string
 	Region             string
 	Bucket             string
 	LocationConstraint bool
+	V2Signature        bool
 	IndexPageSize      int
 	ConnectTimeout     Duration
 	ReadTimeout        Duration
@@ -301,6 +338,10 @@ func (su *URL) UnmarshalText(text []byte) error {
 	u, err := url.Parse(string(text))
 	if err == nil {
 		*su = URL(*u)
+		if su.Path == "" && su.Host != "" {
+			// http://example really means http://example/
+			su.Path = "/"
+		}
 	}
 	return err
 }
diff --git a/sdk/go/arvados/config_test.go b/sdk/go/arvados/config_test.go
index e4d26e03fd..8c77e29287 100644
--- a/sdk/go/arvados/config_test.go
+++ b/sdk/go/arvados/config_test.go
@@ -5,6 +5,8 @@
 package arvados
 
 import (
+	"encoding/json"
+
 	"github.com/ghodss/yaml"
 	check "gopkg.in/check.v1"
 )
@@ -71,3 +73,10 @@ func (s *ConfigSuite) TestInstanceTypeFixup(c *check.C) {
 		c.Check(itm["foo8"].IncludedScratch, check.Equals, ByteSize(0))
 	}
 }
+
+func (s *ConfigSuite) TestURLTrailingSlash(c *check.C) {
+	var a, b map[URL]bool
+	json.Unmarshal([]byte(`{"https://foo.example": true}`), &a)
+	json.Unmarshal([]byte(`{"https://foo.example/": true}`), &b)
+	c.Check(a, check.DeepEquals, b)
+}
diff --git a/sdk/go/arvados/container.go b/sdk/go/arvados/container.go
index a7edddaa33..3d08f2235a 100644
--- a/sdk/go/arvados/container.go
+++ b/sdk/go/arvados/container.go
@@ -28,6 +28,8 @@ type Container struct {
 	SchedulingParameters SchedulingParameters   `json:"scheduling_parameters"`
 	ExitCode             int                    `json:"exit_code"`
 	RuntimeStatus        map[string]interface{} `json:"runtime_status"`
+	StartedAt            *time.Time             `json:"started_at"`  // nil if not yet started
+	FinishedAt           *time.Time             `json:"finished_at"` // nil if not yet finished
 }
 
 // Container is an arvados#container resource.
@@ -106,6 +108,14 @@ type ContainerList struct {
 	Limit          int         `json:"limit"`
 }
 
+// ContainerRequestList is an arvados#containerRequestList resource.
+type ContainerRequestList struct {
+	Items          []ContainerRequest `json:"items"`
+	ItemsAvailable int                `json:"items_available"`
+	Offset         int                `json:"offset"`
+	Limit          int                `json:"limit"`
+}
+
 // ContainerState is a string corresponding to a valid Container state.
 type ContainerState string
 
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 97a62fa7bb..da1710374e 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -6,7 +6,9 @@ package arvados
 
 import (
 	"bufio"
+	"context"
 	"fmt"
+	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
@@ -102,21 +104,57 @@ func (s *KeepService) Mounts(c *Client) ([]KeepMount, error) {
 	return mounts, nil
 }
 
+// Touch updates the timestamp on the given block.
+func (s *KeepService) Touch(ctx context.Context, c *Client, blk string) error {
+	req, err := http.NewRequest("TOUCH", s.url(blk), nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+	}
+	return nil
+}
+
+// Untrash moves/copies the given block out of trash.
+func (s *KeepService) Untrash(ctx context.Context, c *Client, blk string) error {
+	req, err := http.NewRequest("PUT", s.url("untrash/"+blk), nil)
+	if err != nil {
+		return err
+	}
+	resp, err := c.Do(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		body, _ := ioutil.ReadAll(resp.Body)
+		return fmt.Errorf("%s %s: %s", resp.Proto, resp.Status, body)
+	}
+	return nil
+}
+
 // Index returns an unsorted list of blocks at the given mount point.
-func (s *KeepService) IndexMount(c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
-	return s.index(c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
+func (s *KeepService) IndexMount(ctx context.Context, c *Client, mountUUID string, prefix string) ([]KeepServiceIndexEntry, error) {
+	return s.index(ctx, c, s.url("mounts/"+mountUUID+"/blocks?prefix="+prefix))
 }
 
 // Index returns an unsorted list of blocks that can be retrieved from
 // this server.
-func (s *KeepService) Index(c *Client, prefix string) ([]KeepServiceIndexEntry, error) {
-	return s.index(c, s.url("index/"+prefix))
+func (s *KeepService) Index(ctx context.Context, c *Client, prefix string) ([]KeepServiceIndexEntry, error) {
+	return s.index(ctx, c, s.url("index/"+prefix))
 }
 
-func (s *KeepService) index(c *Client, url string) ([]KeepServiceIndexEntry, error) {
-	req, err := http.NewRequest("GET", url, nil)
+func (s *KeepService) index(ctx context.Context, c *Client, url string) ([]KeepServiceIndexEntry, error) {
+	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		return nil, fmt.Errorf("NewRequest(%v): %v", url, err)
+		return nil, fmt.Errorf("NewRequestWithContext(%v): %v", url, err)
 	}
 	resp, err := c.Do(req)
 	if err != nil {
diff --git a/sdk/go/arvados/keep_service_test.go b/sdk/go/arvados/keep_service_test.go
index 8715f74f0b..3a82f4b7ee 100644
--- a/sdk/go/arvados/keep_service_test.go
+++ b/sdk/go/arvados/keep_service_test.go
@@ -5,6 +5,7 @@
 package arvados
 
 import (
+	"context"
 	"net/http"
 
 	check "gopkg.in/check.v1"
@@ -22,6 +23,6 @@ func (*KeepServiceSuite) TestIndexTimeout(c *check.C) {
 		APIHost:   "zzzzz.arvadosapi.com",
 		AuthToken: "xyzzy",
 	}
-	_, err := (&KeepService{}).IndexMount(client, "fake", "")
+	_, err := (&KeepService{}).IndexMount(context.Background(), client, "fake", "")
 	c.Check(err, check.ErrorMatches, `.*timeout.*`)
 }
diff --git a/sdk/go/arvados/link.go b/sdk/go/arvados/link.go
index fbd699f306..fdddfc537d 100644
--- a/sdk/go/arvados/link.go
+++ b/sdk/go/arvados/link.go
@@ -6,14 +6,15 @@ package arvados
 
 // Link is an arvados#link record
 type Link struct {
-	UUID      string `json:"uuid,omiempty"`
-	OwnerUUID string `json:"owner_uuid"`
-	Name      string `json:"name"`
-	LinkClass string `json:"link_class"`
-	HeadUUID  string `json:"head_uuid"`
-	HeadKind  string `json:"head_kind"`
-	TailUUID  string `json:"tail_uuid"`
-	TailKind  string `json:"tail_kind"`
+	UUID       string                 `json:"uuid,omiempty"`
+	OwnerUUID  string                 `json:"owner_uuid"`
+	Name       string                 `json:"name"`
+	LinkClass  string                 `json:"link_class"`
+	HeadUUID   string                 `json:"head_uuid"`
+	HeadKind   string                 `json:"head_kind"`
+	TailUUID   string                 `json:"tail_uuid"`
+	TailKind   string                 `json:"tail_kind"`
+	Properties map[string]interface{} `json:"properties"`
 }
 
 // UserList is an arvados#userList resource.
diff --git a/sdk/go/arvados/virtual_machine.go b/sdk/go/arvados/virtual_machine.go
new file mode 100644
index 0000000000..1506ede209
--- /dev/null
+++ b/sdk/go/arvados/virtual_machine.go
@@ -0,0 +1,25 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+import "time"
+
+// VirtualMachine is an arvados#virtualMachine resource.
+type VirtualMachine struct {
+	UUID               string     `json:"uuid"`
+	OwnerUUID          string     `json:"owner_uuid"`
+	Hostname           string     `json:"hostname"`
+	CreatedAt          *time.Time `json:"created_at"`
+	ModifiedAt         *time.Time `json:"modified_at"`
+	ModifiedByUserUUID string     `json:"modified_by_user_uuid"`
+}
+
+// VirtualMachineList is an arvados#virtualMachineList resource.
+type VirtualMachineList struct {
+	Items          []VirtualMachine `json:"items"`
+	ItemsAvailable int              `json:"items_available"`
+	Offset         int              `json:"offset"`
+	Limit          int              `json:"limit"`
+}
diff --git a/sdk/go/arvadosclient/pool.go b/sdk/go/arvadosclient/pool.go
index 732080770b..bb7867aef7 100644
--- a/sdk/go/arvadosclient/pool.go
+++ b/sdk/go/arvadosclient/pool.go
@@ -6,6 +6,8 @@ package arvadosclient
 
 import (
 	"sync"
+
+	"git.arvados.org/arvados.git/sdk/go/arvados"
 )
 
 // A ClientPool is a pool of ArvadosClients. This is useful for
@@ -14,7 +16,7 @@ import (
 // credentials. See arvados-git-httpd for an example, and sync.Pool
 // for more information about garbage collection.
 type ClientPool struct {
-	// Initialize new clients by coping this one.
+	// Initialize new clients by copying this one.
 	Prototype *ArvadosClient
 
 	pool      *sync.Pool
@@ -25,7 +27,20 @@ type ClientPool struct {
 // MakeClientPool returns a new empty ClientPool, using environment
 // variables to initialize the prototype.
 func MakeClientPool() *ClientPool {
-	proto, err := MakeArvadosClient()
+	return MakeClientPoolWith(nil)
+}
+
+// MakeClientPoolWith returns a new empty ClientPool with a previously
+// initialized arvados.Client.
+func MakeClientPoolWith(client *arvados.Client) *ClientPool {
+	var err error
+	var proto *ArvadosClient
+
+	if client == nil {
+		proto, err = MakeArvadosClient()
+	} else {
+		proto, err = New(client)
+	}
 	return &ClientPool{
 		Prototype: proto,
 		lastErr:   err,
diff --git a/sdk/go/arvadostest/db.go b/sdk/go/arvadostest/db.go
new file mode 100644
index 0000000000..41ecfacc48
--- /dev/null
+++ b/sdk/go/arvadostest/db.go
@@ -0,0 +1,33 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvadostest
+
+import (
+	"context"
+
+	"git.arvados.org/arvados.git/lib/ctrlctx"
+	"git.arvados.org/arvados.git/sdk/go/arvados"
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq"
+	"gopkg.in/check.v1"
+)
+
+// DB returns a DB connection for the given cluster config.
+func DB(c *check.C, cluster *arvados.Cluster) *sqlx.DB {
+	db, err := sqlx.Open("postgres", cluster.PostgreSQL.Connection.String())
+	c.Assert(err, check.IsNil)
+	return db
+}
+
+// TransactionContext returns a context suitable for running a test
+// case in a new transaction, and a rollback func which the caller
+// should call after the test.
+func TransactionContext(c *check.C, db *sqlx.DB) (ctx context.Context, rollback func()) {
+	tx, err := db.Beginx()
+	c.Assert(err, check.IsNil)
+	return ctrlctx.NewWithTransaction(context.Background(), tx), func() {
+		c.Check(tx.Rollback(), check.IsNil)
+	}
+}
diff --git a/sdk/go/keepclient/discover.go b/sdk/go/keepclient/discover.go
index 744ff82685..726c3fb30c 100644
--- a/sdk/go/keepclient/discover.go
+++ b/sdk/go/keepclient/discover.go
@@ -19,7 +19,7 @@ import (
 	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
 )
 
-// ClearCache clears the Keep service discovery cache.
+// RefreshServiceDiscovery clears the Keep service discovery cache.
 func RefreshServiceDiscovery() {
 	var wg sync.WaitGroup
 	defer wg.Wait()
@@ -35,8 +35,8 @@ func RefreshServiceDiscovery() {
 	}
 }
 
-// ClearCacheOnSIGHUP installs a signal handler that calls
-// ClearCache when SIGHUP is received.
+// RefreshServiceDiscoveryOnSIGHUP installs a signal handler that calls
+// RefreshServiceDiscovery when SIGHUP is received.
 func RefreshServiceDiscoveryOnSIGHUP() {
 	svcListCacheMtx.Lock()
 	defer svcListCacheMtx.Unlock()
diff --git a/sdk/go/keepclient/perms.go b/sdk/go/keepclient/perms.go
index a77983322d..23ca7d2f2b 100644
--- a/sdk/go/keepclient/perms.go
+++ b/sdk/go/keepclient/perms.go
@@ -2,108 +2,15 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 
-// Generate and verify permission signatures for Keep locators.
-//
-// See https://dev.arvados.org/projects/arvados/wiki/Keep_locator_format
-
 package keepclient
 
-import (
-	"crypto/hmac"
-	"crypto/sha1"
-	"errors"
-	"fmt"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
-)
+import "git.arvados.org/arvados.git/sdk/go/arvados"
 
 var (
-	// ErrSignatureExpired - a signature was rejected because the
-	// expiry time has passed.
-	ErrSignatureExpired = errors.New("Signature expired")
-	// ErrSignatureInvalid - a signature was rejected because it
-	// was badly formatted or did not match the given secret key.
-	ErrSignatureInvalid = errors.New("Invalid signature")
-	// ErrSignatureMissing - the given locator does not have a
-	// signature hint.
-	ErrSignatureMissing = errors.New("Missing signature")
+	ErrSignatureExpired = arvados.ErrSignatureExpired
+	ErrSignatureInvalid = arvados.ErrSignatureInvalid
+	ErrSignatureMissing = arvados.ErrSignatureMissing
+	SignLocator         = arvados.SignLocator
+	SignedLocatorRe     = arvados.SignedLocatorRe
+	VerifySignature     = arvados.VerifySignature
 )
-
-// makePermSignature generates a SHA-1 HMAC digest for the given blob,
-// token, expiry, and site secret.
-func makePermSignature(blobHash, apiToken, expiry, blobSignatureTTL string, permissionSecret []byte) string {
-	hmac := hmac.New(sha1.New, permissionSecret)
-	hmac.Write([]byte(blobHash))
-	hmac.Write([]byte("@"))
-	hmac.Write([]byte(apiToken))
-	hmac.Write([]byte("@"))
-	hmac.Write([]byte(expiry))
-	hmac.Write([]byte("@"))
-	hmac.Write([]byte(blobSignatureTTL))
-	digest := hmac.Sum(nil)
-	return fmt.Sprintf("%x", digest)
-}
-
-// SignLocator returns blobLocator with a permission signature
-// added. If either permissionSecret or apiToken is empty, blobLocator
-// is returned untouched.
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func SignLocator(blobLocator, apiToken string, expiry time.Time, blobSignatureTTL time.Duration, permissionSecret []byte) string {
-	if len(permissionSecret) == 0 || apiToken == "" {
-		return blobLocator
-	}
-	// Strip off all hints: only the hash is used to sign.
-	blobHash := strings.Split(blobLocator, "+")[0]
-	timestampHex := fmt.Sprintf("%08x", expiry.Unix())
-	blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
-	return blobLocator +
-		"+A" + makePermSignature(blobHash, apiToken, timestampHex, blobSignatureTTLHex, permissionSecret) +
-		"@" + timestampHex
-}
-
-var SignedLocatorRe = regexp.MustCompile(
-	//1                 2          34                         5   6                  7                 89
-	`^([[:xdigit:]]{32})(\+[0-9]+)?((\+[B-Z][A-Za-z0-9@_-]*)*)(\+A([[:xdigit:]]{40})@([[:xdigit:]]{8}))((\+[B-Z][A-Za-z0-9@_-]*)*)$`)
-
-// VerifySignature returns nil if the signature on the signedLocator
-// can be verified using the given apiToken. Otherwise it returns
-// ErrSignatureExpired (if the signature's expiry time has passed,
-// which is something the client could have figured out
-// independently), ErrSignatureMissing (if there is no signature hint
-// at all), or ErrSignatureInvalid (if the signature is present but
-// badly formatted or incorrect).
-//
-// This function is intended to be used by system components and admin
-// utilities: userland programs do not know the permissionSecret.
-func VerifySignature(signedLocator, apiToken string, blobSignatureTTL time.Duration, permissionSecret []byte) error {
-	matches := SignedLocatorRe.FindStringSubmatch(signedLocator)
-	if matches == nil {
-		return ErrSignatureMissing
-	}
-	blobHash := matches[1]
-	signatureHex := matches[6]
-	expiryHex := matches[7]
-	if expiryTime, err := parseHexTimestamp(expiryHex); err != nil {
-		return ErrSignatureInvalid
-	} else if expiryTime.Before(time.Now()) {
-		return ErrSignatureExpired
-	}
-	blobSignatureTTLHex := strconv.FormatInt(int64(blobSignatureTTL.Seconds()), 16)
-	if signatureHex != makePermSignature(blobHash, apiToken, expiryHex, blobSignatureTTLHex, permissionSecret) {
-		return ErrSignatureInvalid
-	}
-	return nil
-}
-
-func parseHexTimestamp(timestampHex string) (ts time.Time, err error) {
-	if tsInt, e := strconv.ParseInt(timestampHex, 16, 0); e == nil {
-		ts = time.Unix(tsInt, 0)
-	} else {
-		err = e
-	}
-	return ts, err
-}
diff --git a/sdk/go/keepclient/perms_test.go b/sdk/go/keepclient/perms_test.go
deleted file mode 100644
index f8107f4696..0000000000
--- a/sdk/go/keepclient/perms_test.go
+++ /dev/null
@@ -1,103 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: Apache-2.0
-
-package keepclient
-
-import (
-	"testing"
-	"time"
-)
-
-const (
-	knownHash    = "acbd18db4cc2f85cedef654fccc4a4d8"
-	knownLocator = knownHash + "+3"
-	knownToken   = "hocfupkn2pjhrpgp2vxv8rsku7tvtx49arbc9s4bvu7p7wxqvk"
-	knownKey     = "13u9fkuccnboeewr0ne3mvapk28epf68a3bhj9q8sb4l6e4e5mkk" +
-		"p6nhj2mmpscgu1zze5h5enydxfe3j215024u16ij4hjaiqs5u4pzsl3nczmaoxnc" +
-		"ljkm4875xqn4xv058koz3vkptmzhyheiy6wzevzjmdvxhvcqsvr5abhl15c2d4o4" +
-		"jhl0s91lojy1mtrzqqvprqcverls0xvy9vai9t1l1lvvazpuadafm71jl4mrwq2y" +
-		"gokee3eamvjy8qq1fvy238838enjmy5wzy2md7yvsitp5vztft6j4q866efym7e6" +
-		"vu5wm9fpnwjyxfldw3vbo01mgjs75rgo7qioh8z8ij7jpyp8508okhgbbex3ceei" +
-		"786u5rw2a9gx743dj3fgq2irk"
-	knownSignature     = "89118b78732c33104a4d6231e8b5a5fa1e4301e3"
-	knownTimestamp     = "7fffffff"
-	knownSigHint       = "+A" + knownSignature + "@" + knownTimestamp
-	knownSignedLocator = knownLocator + knownSigHint
-	blobSignatureTTL   = 1209600 * time.Second
-)
-
-func TestSignLocator(t *testing.T) {
-	if ts, err := parseHexTimestamp(knownTimestamp); err != nil {
-		t.Errorf("bad knownTimestamp %s", knownTimestamp)
-	} else {
-		if knownSignedLocator != SignLocator(knownLocator, knownToken, ts, blobSignatureTTL, []byte(knownKey)) {
-			t.Fail()
-		}
-	}
-}
-
-func TestVerifySignature(t *testing.T) {
-	if VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
-		t.Fail()
-	}
-}
-
-func TestVerifySignatureExtraHints(t *testing.T) {
-	if VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
-		t.Fatal("Verify cannot handle hint before permission signature")
-	}
-
-	if VerifySignature(knownLocator+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
-		t.Fatal("Verify cannot handle hint after permission signature")
-	}
-
-	if VerifySignature(knownLocator+"+K@xyzzy"+knownSigHint+"+Zfoo", knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
-		t.Fatal("Verify cannot handle hints around permission signature")
-	}
-}
-
-// The size hint on the locator string should not affect signature validation.
-func TestVerifySignatureWrongSize(t *testing.T) {
-	if VerifySignature(knownHash+"+999999"+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
-		t.Fatal("Verify cannot handle incorrect size hint")
-	}
-
-	if VerifySignature(knownHash+knownSigHint, knownToken, blobSignatureTTL, []byte(knownKey)) != nil {
-		t.Fatal("Verify cannot handle missing size hint")
-	}
-}
-
-func TestVerifySignatureBadSig(t *testing.T) {
-	badLocator := knownLocator + "+Aaaaaaaaaaaaaaaa@" + knownTimestamp
-	if VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureMissing {
-		t.Fail()
-	}
-}
-
-func TestVerifySignatureBadTimestamp(t *testing.T) {
-	badLocator := knownLocator + "+A" + knownSignature + "@OOOOOOOl"
-	if VerifySignature(badLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureMissing {
-		t.Fail()
-	}
-}
-
-func TestVerifySignatureBadSecret(t *testing.T) {
-	if VerifySignature(knownSignedLocator, knownToken, blobSignatureTTL, []byte("00000000000000000000")) != ErrSignatureInvalid {
-		t.Fail()
-	}
-}
-
-func TestVerifySignatureBadToken(t *testing.T) {
-	if VerifySignature(knownSignedLocator, "00000000", blobSignatureTTL, []byte(knownKey)) != ErrSignatureInvalid {
-		t.Fail()
-	}
-}
-
-func TestVerifySignatureExpired(t *testing.T) {
-	yesterday := time.Now().AddDate(0, 0, -1)
-	expiredLocator := SignLocator(knownHash, knownToken, yesterday, blobSignatureTTL, []byte(knownKey))
-	if VerifySignature(expiredLocator, knownToken, blobSignatureTTL, []byte(knownKey)) != ErrSignatureExpired {
-		t.Fail()
-	}
-}
diff --git a/sdk/python/arvados/commands/federation_migrate.py b/sdk/python/arvados/commands/federation_migrate.py
index 445775cced..5c1bb29e76 100755
--- a/sdk/python/arvados/commands/federation_migrate.py
+++ b/sdk/python/arvados/commands/federation_migrate.py
@@ -22,6 +22,7 @@ import hmac
 import urllib.parse
 import os
 import hashlib
+import re
 from arvados._version import __version__
 
 EMAIL=0
@@ -169,19 +170,20 @@ def read_migrations(args, by_email, by_username):
 
 def update_username(args, email, user_uuid, username, migratecluster, migratearv):
     print("(%s) Updating username of %s to '%s' on %s" % (email, user_uuid, username, migratecluster))
-    if not args.dry_run:
-        try:
-            conflicts = migratearv.users().list(filters=[["username", "=", username]], bypass_federation=True).execute()
-            if conflicts["items"]:
-                # There's already a user with the username, move the old user out of the way
-                migratearv.users().update(uuid=conflicts["items"][0]["uuid"],
-                                          bypass_federation=True,
-                                          body={"user": {"username": username+"migrate"}}).execute()
-            migratearv.users().update(uuid=user_uuid,
-                                      bypass_federation=True,
-                                      body={"user": {"username": username}}).execute()
-        except arvados.errors.ApiError as e:
-            print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, user_uuid, username, migratecluster, e))
+    if args.dry_run:
+        return
+    try:
+        conflicts = migratearv.users().list(filters=[["username", "=", username]], bypass_federation=True).execute()
+        if conflicts["items"]:
+            # There's already a user with the username, move the old user out of the way
+            migratearv.users().update(uuid=conflicts["items"][0]["uuid"],
+                                        bypass_federation=True,
+                                        body={"user": {"username": username+"migrate"}}).execute()
+        migratearv.users().update(uuid=user_uuid,
+                                    bypass_federation=True,
+                                    body={"user": {"username": username}}).execute()
+    except arvados.errors.ApiError as e:
+        print("(%s) Error updating username of %s to '%s' on %s: %s" % (email, user_uuid, username, migratecluster, e))
 
 
 def choose_new_user(args, by_email, email, userhome, username, old_user_uuid, clusters):
@@ -212,11 +214,17 @@ def choose_new_user(args, by_email, email, userhome, username, old_user_uuid, cl
                 conflicts = homearv.users().list(filters=[["username", "=", username]],
                                                  bypass_federation=True).execute()
                 if conflicts["items"]:
-                    homearv.users().update(uuid=conflicts["items"][0]["uuid"],
-                                           bypass_federation=True,
-                                           body={"user": {"username": username+"migrate"}}).execute()
-                user = homearv.users().create(body={"user": {"email": email, "username": username,
-                                                             "is_active": olduser["is_active"]}}).execute()
+                    homearv.users().update(
+                        uuid=conflicts["items"][0]["uuid"],
+                        bypass_federation=True,
+                        body={"user": {"username": username+"migrate"}}).execute()
+                user = homearv.users().create(
+                    body={"user": {
+                        "email": email,
+                        "first_name": olduser["first_name"],
+                        "last_name": olduser["last_name"],
+                        "username": username,
+                        "is_active": olduser["is_active"]}}).execute()
             except arvados.errors.ApiError as e:
                 print("(%s) Could not create user: %s" % (email, str(e)))
                 return None
@@ -271,7 +279,7 @@ def activate_remote_user(args, email, homearv, migratearv, old_user_uuid, new_us
             newuser = arvados.api(host=ru.netloc, token=salted,
                                   insecure=os.environ.get("ARVADOS_API_HOST_INSECURE")).users().current().execute()
         else:
-            newuser = {"is_active": True, "username": username}
+            newuser = {"is_active": True, "username": email.split('@')[0], "is_admin": False}
     except arvados.errors.ApiError as e:
         print("(%s) Error getting user info for %s from %s: %s" % (email, new_user_uuid, migratecluster, e))
         return None
@@ -287,39 +295,48 @@ def activate_remote_user(args, email, homearv, migratearv, old_user_uuid, new_us
             return None
 
     if olduser["is_admin"] and not newuser["is_admin"]:
-        print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s" % (email, old_user_uuid, new_user_uuid, migratecluster))
+        print("(%s) Not migrating %s because user is admin but target user %s is not admin on %s. Please ensure the user admin status is the same on both clusters. Note that a federated admin account has admin privileges on the entire federation." % (email, old_user_uuid, new_user_uuid, migratecluster))
         return None
 
     return newuser
 
 def migrate_user(args, migratearv, email, new_user_uuid, old_user_uuid):
+    if args.dry_run:
+        return
     try:
-        if not args.dry_run:
+        new_owner_uuid = new_user_uuid
+        if args.data_into_subproject:
             grp = migratearv.groups().create(body={
                 "owner_uuid": new_user_uuid,
                 "name": "Migrated from %s (%s)" % (email, old_user_uuid),
                 "group_class": "project"
             }, ensure_unique_name=True).execute()
-            migratearv.users().merge(old_user_uuid=old_user_uuid,
-                                     new_user_uuid=new_user_uuid,
-                                     new_owner_uuid=grp["uuid"],
-                                     redirect_to_new_user=True).execute()
+            new_owner_uuid = grp["uuid"]
+        migratearv.users().merge(old_user_uuid=old_user_uuid,
+                                    new_user_uuid=new_user_uuid,
+                                    new_owner_uuid=new_owner_uuid,
+                                    redirect_to_new_user=True).execute()
     except arvados.errors.ApiError as e:
-        print("(%s) Error migrating user: %s" % (email, e))
+        name_collision = re.search(r'Key \(owner_uuid, name\)=\((.*?), (.*?)\) already exists\.\n.*UPDATE "(.*?)"', e._get_reason())
+        if name_collision:
+            target_owner, rsc_name, rsc_type = name_collision.groups()
+            print("(%s) Cannot migrate to %s because both origin and target users have a %s named '%s'. Please rename the conflicting items or use --data-into-subproject to migrate all users' data into a special subproject." % (email, target_owner, rsc_type[:-1], rsc_name))
+        else:
+            print("(%s) Skipping user migration because of error: %s" % (email, e))
 
 
 def main():
-
     parser = argparse.ArgumentParser(description='Migrate users to federated identity, see https://doc.arvados.org/admin/merge-remote-account.html')
     parser.add_argument(
         '--version', action='version', version="%s %s" % (sys.argv[0], __version__),
         help='Print version and exit.')
-    parser.add_argument('--tokens', type=str, required=False)
+    parser.add_argument('--tokens', type=str, metavar='FILE', required=False, help="Read tokens from FILE. Not needed when using LoginCluster.")
+    parser.add_argument('--data-into-subproject', action="store_true", help="Migrate user's data into a separate subproject. This can be used to avoid name collisions from within an account.")
     group = parser.add_mutually_exclusive_group(required=True)
-    group.add_argument('--report', type=str, help="Generate report .csv file listing users by email address and their associated Arvados accounts")
-    group.add_argument('--migrate', type=str, help="Consume report .csv and migrate users to designated Arvados accounts")
-    group.add_argument('--dry-run', type=str, help="Consume report .csv and report how user would be migrated to designated Arvados accounts")
-    group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected")
+    group.add_argument('--report', type=str, metavar='FILE', help="Generate report .csv file listing users by email address and their associated Arvados accounts.")
+    group.add_argument('--migrate', type=str, metavar='FILE', help="Consume report .csv and migrate users to designated Arvados accounts.")
+    group.add_argument('--dry-run', type=str, metavar='FILE', help="Consume report .csv and report how user would be migrated to designated Arvados accounts.")
+    group.add_argument('--check', action="store_true", help="Check that tokens are usable and the federation is well connected.")
     args = parser.parse_args()
 
     clusters, errors, loginCluster = connect_clusters(args)
diff --git a/sdk/python/arvados/keep.py b/sdk/python/arvados/keep.py
index 86a28f54c4..bc43b849c3 100644
--- a/sdk/python/arvados/keep.py
+++ b/sdk/python/arvados/keep.py
@@ -375,6 +375,8 @@ class KeepClient(object):
                     curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
                     if self.insecure:
                         curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+                    else:
+                        curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
                     if method == "HEAD":
                         curl.setopt(pycurl.NOBODY, True)
                     self._setcurltimeouts(curl, timeout, method=="HEAD")
@@ -473,6 +475,8 @@ class KeepClient(object):
                     curl.setopt(pycurl.HEADERFUNCTION, self._headerfunction)
                     if self.insecure:
                         curl.setopt(pycurl.SSL_VERIFYPEER, 0)
+                    else:
+                        curl.setopt(pycurl.CAINFO, arvados.util.ca_certs_path())
                     self._setcurltimeouts(curl, timeout)
                     try:
                         curl.perform()
diff --git a/sdk/python/arvados/util.py b/sdk/python/arvados/util.py
index dcc0417c13..6c9822e9f0 100644
--- a/sdk/python/arvados/util.py
+++ b/sdk/python/arvados/util.py
@@ -396,6 +396,9 @@ def ca_certs_path(fallback=httplib2.CA_CERTS):
     it returns the value of `fallback` (httplib2's CA certs by default).
     """
     for ca_certs_path in [
+        # SSL_CERT_FILE and SSL_CERT_DIR are openssl overrides - note
+        # that httplib2 itself also supports HTTPLIB2_CA_CERTS.
+        os.environ.get('SSL_CERT_FILE'),
         # Arvados specific:
         '/etc/arvados/ca-certificates.crt',
         # Debian:
@@ -403,7 +406,7 @@ def ca_certs_path(fallback=httplib2.CA_CERTS):
         # Red Hat:
         '/etc/pki/tls/certs/ca-bundle.crt',
         ]:
-        if os.path.exists(ca_certs_path):
+        if ca_certs_path and os.path.exists(ca_certs_path):
             return ca_certs_path
     return fallback
 
diff --git a/sdk/python/setup.py b/sdk/python/setup.py
index a726b49fe3..589533177a 100644
--- a/sdk/python/setup.py
+++ b/sdk/python/setup.py
@@ -54,6 +54,7 @@ setup(name='arvados-python-client',
           'ruamel.yaml >=0.15.54, <=0.16.5',
           'setuptools',
           'ws4py >=0.4.2',
+          'rsa < 4.1'
       ],
       extras_require={
           ':os.name=="posix" and python_version<"3"': ['subprocess32 >= 3.5.1'],
diff --git a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
index aa859cba4f..cc0fb0fe06 100644
--- a/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
+++ b/sdk/python/tests/fed-migrate/arvbox-make-federation.cwl
@@ -9,6 +9,8 @@ inputs:
   branch:
     type: string
     default: master
+  arvbox_mode:
+    type: string?
 outputs:
   arvados_api_hosts:
     type: string[]
@@ -39,6 +41,7 @@ steps:
     in:
       arvbox_base: arvbox_base
       branch: branch
+      arvbox_mode: arvbox_mode
       logincluster:
         default: true
     out: [arvados_api_hosts, arvados_cluster_ids, arvado_api_host_insecure, superuser_tokens, arvbox_containers, arvbox_bin]
diff --git a/sdk/python/tests/fed-migrate/check.py b/sdk/python/tests/fed-migrate/check.py
index a2c0096165..c231cc0735 100644
--- a/sdk/python/tests/fed-migrate/check.py
+++ b/sdk/python/tests/fed-migrate/check.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import arvados
 import json
 import sys
@@ -21,7 +25,7 @@ def check_A(users):
     for i in range(1, 10):
         found = False
         for u in users["items"]:
-            if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i):
+            if u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser":
                 found = True
                 by_username[u["username"]] = u["uuid"]
         assert found
@@ -60,6 +64,7 @@ for i in range(2, 9):
     found = False
     for u in users["items"]:
         if (u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and
+            u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser" and
             u["uuid"] == by_username[u["username"]] and u["is_active"] is True):
             found = True
     assert found, "Not found case%i" % i
@@ -67,6 +72,7 @@ for i in range(2, 9):
 found = False
 for u in users["items"]:
     if (u["username"] == "case9" and u["email"] == "case9@test" and
+        u["first_name"] == "Case9" and u["last_name"] == "Testuser" and
         u["uuid"] == by_username[u["username"]] and u["is_active"] is False):
         found = True
 assert found
@@ -87,6 +93,7 @@ for i in (2, 4, 6, 7, 8):
     found = False
     for u in users["items"]:
         if (u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and
+            u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser" and
             u["uuid"] == by_username[u["username"]] and u["is_active"] is True):
             found = True
     assert found
@@ -97,6 +104,7 @@ for i in (3, 5, 9):
     found = False
     for u in users["items"]:
         if (u["username"] == ("case%d" % i) and u["email"] == ("case%d@test" % i) and
+            u["first_name"] == ("Case%d" % i) and u["last_name"] == "Testuser" and
             u["uuid"] == by_username[u["username"]] and u["is_active"] is True):
             found = True
     assert not found
diff --git a/sdk/python/tests/fed-migrate/create_users.py b/sdk/python/tests/fed-migrate/create_users.py
index cea624ec4c..0b5732293d 100644
--- a/sdk/python/tests/fed-migrate/create_users.py
+++ b/sdk/python/tests/fed-migrate/create_users.py
@@ -1,3 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
 import arvados
 import json
 import sys
@@ -11,13 +15,21 @@ apiC = arvados.api(host=j["arvados_api_hosts"][2], token=j["superuser_tokens"][2
 def maketoken(newtok):
     return 'v2/' + newtok["uuid"] + '/' + newtok["api_token"]
 
+def get_user_data(case_nr, is_active=True):
+    return {
+        "email": "case{}@test".format(case_nr),
+        "first_name": "Case{}".format(case_nr),
+        "last_name": "Testuser",
+        "is_active": is_active
+    }
+
 # case 1
 # user only exists on cluster A
-apiA.users().create(body={"user": {"email": "case1@test", "is_active": True}}).execute()
+apiA.users().create(body={"user": get_user_data(case_nr=1)}).execute()
 
 # case 2
 # user exists on cluster A and has remotes on B and C
-case2 = apiA.users().create(body={"user": {"email": "case2@test", "is_active": True}}).execute()
+case2 = apiA.users().create(body={"user": get_user_data(case_nr=2)}).execute()
 newtok = apiA.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case2["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtok), insecure=True).users().current().execute()
@@ -25,11 +37,11 @@ arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=Tr
 
 # case 3
 # user only exists on cluster B
-case3 = apiB.users().create(body={"user": {"email": "case3@test", "is_active": True}}).execute()
+case3 = apiB.users().create(body={"user": get_user_data(case_nr=3)}).execute()
 
 # case 4
 # user only exists on cluster B and has remotes on A and C
-case4 = apiB.users().create(body={"user": {"email": "case4@test", "is_active": True}}).execute()
+case4 = apiB.users().create(body={"user": get_user_data(case_nr=4)}).execute()
 newtok = apiB.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case4["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtok), insecure=True).users().current().execute()
@@ -38,18 +50,18 @@ arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtok), insecure=Tr
 
 # case 5
 # user exists on both cluster A and B
-case5 = apiA.users().create(body={"user": {"email": "case5@test", "is_active": True}}).execute()
-case5 = apiB.users().create(body={"user": {"email": "case5@test", "is_active": True}}).execute()
+case5 = apiA.users().create(body={"user": get_user_data(case_nr=5)}).execute()
+case5 = apiB.users().create(body={"user": get_user_data(case_nr=5)}).execute()
 
 # case 6
 # user exists on both cluster A and B, with remotes on A, B and C
-case6_A = apiA.users().create(body={"user": {"email": "case6@test", "is_active": True}}).execute()
+case6_A = apiA.users().create(body={"user": get_user_data(case_nr=6)}).execute()
 newtokA = apiA.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case6_A["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
 arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=True).users().current().execute()
 
-case6_B = apiB.users().create(body={"user": {"email": "case6@test", "is_active": True}}).execute()
+case6_B = apiB.users().create(body={"user": get_user_data(case_nr=6)}).execute()
 newtokB = apiB.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case6_B["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
@@ -57,13 +69,13 @@ arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=T
 
 # case 7
 # user exists on both cluster B and A, with remotes on A, B and C
-case7_B = apiB.users().create(body={"user": {"email": "case7@test", "is_active": True}}).execute()
+case7_B = apiB.users().create(body={"user": get_user_data(case_nr=7)}).execute()
 newtokB = apiB.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case7_B["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
 arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
 
-case7_A = apiA.users().create(body={"user": {"email": "case7@test", "is_active": True}}).execute()
+case7_A = apiA.users().create(body={"user": get_user_data(case_nr=7)}).execute()
 newtokA = apiA.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case7_A["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokA), insecure=True).users().current().execute()
@@ -71,13 +83,13 @@ arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokA), insecure=T
 
 # case 8
 # user exists on both cluster B and C, with remotes on A, B and C
-case8_B = apiB.users().create(body={"user": {"email": "case8@test", "is_active": True}}).execute()
+case8_B = apiB.users().create(body={"user": get_user_data(case_nr=8)}).execute()
 newtokB = apiB.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case8_B["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokB), insecure=True).users().current().execute()
 arvados.api(host=j["arvados_api_hosts"][2], token=maketoken(newtokB), insecure=True).users().current().execute()
 
-case8_C = apiC.users().create(body={"user": {"email": "case8@test", "is_active": True}}).execute()
+case8_C = apiC.users().create(body={"user": get_user_data(case_nr=8)}).execute()
 newtokC = apiC.api_client_authorizations().create(body={
     "api_client_authorization": {'owner_uuid': case8_C["uuid"]}}).execute()
 arvados.api(host=j["arvados_api_hosts"][0], token=maketoken(newtokC), insecure=True).users().current().execute()
@@ -85,4 +97,4 @@ arvados.api(host=j["arvados_api_hosts"][1], token=maketoken(newtokC), insecure=T
 
 # case 9
 # user only exists on cluster B, but is inactive
-case9 = apiB.users().create(body={"user": {"email": "case9@test", "is_active": False}}).execute()
+case9 = apiB.users().create(body={"user": get_user_data(case_nr=9, is_active=False)}).execute()
diff --git a/sdk/python/tests/fed-migrate/jenkins.sh b/sdk/python/tests/fed-migrate/jenkins.sh
new file mode 100755
index 0000000000..e5dd8aa913
--- /dev/null
+++ b/sdk/python/tests/fed-migrate/jenkins.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+if test -z "$WORKSPACE" ; then
+    echo "WORKSPACE unset"
+    exit 1
+fi
+
+docker stop fedbox1 fedbox2 fedbox3
+docker rm fedbox1 fedbox2 fedbox3
+docker rm fedbox1-data fedbox2-data fedbox3-data
+
+set -ex
+
+mkdir -p $WORKSPACE/tmp
+cd $WORKSPACE/tmp
+virtualenv --python python3 venv3
+. venv3/bin/activate
+
+cd $WORKSPACE/sdk/python
+pip install -e .
+
+cd $WORKSPACE/sdk/cwl
+pip install -e .
+
+export PATH=$PATH:$WORKSPACE/tools/arvbox/bin
+
+mkdir -p $WORKSPACE/tmp/arvbox
+cd $WORKSPACE/sdk/python/tests/fed-migrate
+cwltool arvbox-make-federation.cwl \
+	--arvbox_base $WORKSPACE/tmp/arvbox \
+	--branch $(git rev-parse HEAD) \
+	--arvbox_mode localdemo > fed.json
+
+cwltool fed-migrate.cwl fed.json
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 6e872a615c..85b4f5b37b 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -71,6 +71,25 @@ http {
       proxy_request_buffering off;
     }
   }
+  upstream health {
+    server {{LISTENHOST}}:{{HEALTHPORT}};
+  }
+  server {
+    listen {{LISTENHOST}}:{{HEALTHSSLPORT}} ssl default_server;
+    server_name health;
+    ssl_certificate "{{SSLCERT}}";
+    ssl_certificate_key "{{SSLKEY}}";
+    location  / {
+      proxy_pass http://health;
+      proxy_set_header Host $http_host;
+      proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+      proxy_set_header X-Forwarded-Proto https;
+      proxy_redirect off;
+
+      proxy_http_version 1.1;
+      proxy_request_buffering off;
+    }
+  }
   server {
     listen {{LISTENHOST}}:{{KEEPWEBDLSSLPORT}} ssl default_server;
     server_name keep-web-dl ~.*;
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 734bb04270..fe32547fcb 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -615,6 +615,8 @@ def run_nginx():
     nginxconf['KEEPPROXYSSLPORT'] = external_port_from_config("Keepproxy")
     nginxconf['GITPORT'] = internal_port_from_config("GitHTTP")
     nginxconf['GITSSLPORT'] = external_port_from_config("GitHTTP")
+    nginxconf['HEALTHPORT'] = internal_port_from_config("Health")
+    nginxconf['HEALTHSSLPORT'] = external_port_from_config("Health")
     nginxconf['WSPORT'] = internal_port_from_config("Websocket")
     nginxconf['WSSSLPORT'] = external_port_from_config("Websocket")
     nginxconf['WORKBENCH1PORT'] = internal_port_from_config("Workbench1")
@@ -654,6 +656,8 @@ def setup_config():
     workbench1_external_port = find_available_port()
     git_httpd_port = find_available_port()
     git_httpd_external_port = find_available_port()
+    health_httpd_port = find_available_port()
+    health_httpd_external_port = find_available_port()
     keepproxy_port = find_available_port()
     keepproxy_external_port = find_available_port()
     keepstore_ports = sorted([str(find_available_port()) for _ in xrange(0,4)])
@@ -709,6 +713,12 @@ def setup_config():
                 "http://%s:%s"%(localhost, git_httpd_port): {}
             },
         },
+        "Health": {
+            "ExternalURL": "https://%s:%s" % (localhost, health_httpd_external_port),
+            "InternalURLs": {
+                "http://%s:%s"%(localhost, health_httpd_port): {}
+            },
+        },
         "Keepstore": {
             "InternalURLs": {
                 "http://%s:%s"%(localhost, port): {} for port in keepstore_ports
@@ -747,8 +757,10 @@ def setup_config():
                     "RailsSessionSecretToken": "e24205c490ac07e028fd5f8a692dcb398bcd654eff1aef5f9fe6891994b18483",
                 },
                 "Login": {
-                    "ProviderAppID": "arvados-server",
-                    "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
+                    "SSO": {
+                        "ProviderAppID": "arvados-server",
+                        "ProviderAppSecret": "608dbf356a327e2d0d4932b60161e212c2d8d8f5e25690d7b622f850a990cd33",
+                    },
                 },
                 "SystemLogs": {
                     "LogLevel": ('info' if os.environ.get('ARVADOS_DEBUG', '') in ['','0'] else 'debug'),
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 24d5ad5b64..127a09ee2d 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -180,7 +180,7 @@ GEM
     pg (1.1.4)
     power_assert (1.1.4)
     public_suffix (4.0.3)
-    rack (2.0.7)
+    rack (2.2.3)
     rack-test (0.6.3)
       rack (>= 1.0)
     rails (5.0.7.2)
@@ -273,7 +273,7 @@ GEM
       json (>= 1.8.0)
     websocket-driver (0.6.5)
       websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.3)
+    websocket-extensions (0.1.5)
 
 PLATFORMS
   ruby
@@ -317,4 +317,4 @@ DEPENDENCIES
   uglifier (~> 2.0)
 
 BUNDLED WITH
-   1.11
+   1.16.6
diff --git a/services/api/app/controllers/arvados/v1/users_controller.rb b/services/api/app/controllers/arvados/v1/users_controller.rb
index d9ab5556ff..867b9a6e6a 100644
--- a/services/api/app/controllers/arvados/v1/users_controller.rb
+++ b/services/api/app/controllers/arvados/v1/users_controller.rb
@@ -54,7 +54,11 @@ class Arvados::V1::UsersController < ApplicationController
       @object = current_user
     end
     if not @object.is_active
-      if not (current_user.is_admin or @object.is_invited)
+      if @object.uuid[0..4] == Rails.configuration.Login.LoginCluster &&
+         @object.uuid[0..4] != Rails.configuration.ClusterID
+        logger.warn "Local user #{@object.uuid} called users#activate but only LoginCluster can do that"
+        raise ArgumentError.new "cannot activate user #{@object.uuid} here, only the #{@object.uuid[0..4]} cluster can do that"
+      elsif not (current_user.is_admin or @object.is_invited)
         logger.warn "User #{@object.uuid} called users.activate " +
           "but is not invited"
         raise ArgumentError.new "Cannot activate without being invited."
diff --git a/services/api/app/controllers/database_controller.rb b/services/api/app/controllers/database_controller.rb
index d6045a5dcb..5c4cf7bc16 100644
--- a/services/api/app/controllers/database_controller.rb
+++ b/services/api/app/controllers/database_controller.rb
@@ -75,9 +75,10 @@ class DatabaseController < ApplicationController
       raise
     end
 
-    require 'refresh_permission_view'
+    require 'update_permissions'
 
-    refresh_permission_view
+    refresh_permissions
+    refresh_trashed
 
     # Done.
     send_json success: true
diff --git a/services/api/app/controllers/user_sessions_controller.rb b/services/api/app/controllers/user_sessions_controller.rb
index 85f32772b1..582b98cf2d 100644
--- a/services/api/app/controllers/user_sessions_controller.rb
+++ b/services/api/app/controllers/user_sessions_controller.rb
@@ -89,7 +89,7 @@ class UserSessionsController < ApplicationController
 
     flash[:notice] = 'You have logged off'
     return_to = params[:return_to] || root_url
-    redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}/users/sign_out?redirect_uri=#{CGI.escape return_to}"
+    redirect_to "#{Rails.configuration.Services.SSO.ExternalURL}users/sign_out?redirect_uri=#{CGI.escape return_to}"
   end
 
   # login - Just bounce to /auth/joshid. The only purpose of this function is
diff --git a/services/api/app/models/api_client_authorization.rb b/services/api/app/models/api_client_authorization.rb
index 5386cb119a..6057c4d269 100644
--- a/services/api/app/models/api_client_authorization.rb
+++ b/services/api/app/models/api_client_authorization.rb
@@ -164,6 +164,9 @@ class ApiClientAuthorization < ArvadosModel
          (secret == auth.api_token ||
           secret == OpenSSL::HMAC.hexdigest('sha1', auth.api_token, remote))
         # found it
+        if token_uuid[0..4] != Rails.configuration.ClusterID
+          Rails.logger.debug "found cached remote token #{token_uuid} with secret #{secret} in local db"
+        end
         return auth
       end
 
@@ -274,6 +277,7 @@ class ApiClientAuthorization < ArvadosModel
                                 api_token: secret,
                                 api_client_id: 0,
                                 expires_at: Time.now + Rails.configuration.Login.RemoteTokenRefresh)
+        Rails.logger.debug "cached remote token #{token_uuid} with secret #{secret} in local db"
       end
       return auth
     else
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 816dbf4758..01a31adb91 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -285,10 +285,13 @@ class ArvadosModel < ApplicationRecord
     sql_conds = nil
     user_uuids = users_list.map { |u| u.uuid }
 
+    # For details on how the trashed_groups table is constructed, see
+    # see db/migrate/20200501150153_permission_table.rb
+
     exclude_trashed_records = ""
     if !include_trash and (sql_table == "groups" or sql_table == "collections") then
-      # Only include records that are not explicitly trashed
-      exclude_trashed_records = "AND #{sql_table}.is_trashed = false"
+      # Only include records that are not trashed
+      exclude_trashed_records = "AND (#{sql_table}.trash_at is NULL or #{sql_table}.trash_at > statement_timestamp())"
     end
 
     if users_list.select { |u| u.is_admin }.any?
@@ -296,16 +299,28 @@ class ArvadosModel < ApplicationRecord
       if !include_trash
         if sql_table != "api_client_authorizations"
           # Only include records where the owner is not trashed
-          sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
-                      "WHERE trashed = 1) #{exclude_trashed_records}"
+          sql_conds = "#{sql_table}.owner_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} "+
+                      "where trash_at <= statement_timestamp()) #{exclude_trashed_records}"
         end
       end
     else
       trashed_check = ""
       if !include_trash then
-        trashed_check = "AND trashed = 0"
+        trashed_check = "AND target_uuid NOT IN (SELECT group_uuid FROM #{TRASHED_GROUPS} where trash_at <= statement_timestamp())"
       end
 
+      # The core of the permission check is a join against the
+      # materialized_permissions table to determine if the user has at
+      # least read permission to either the object itself or its
+      # direct owner (if traverse_owned is true).  See
+      # db/migrate/20200501150153_permission_table.rb for details on
+      # how the permissions are computed.
+
+      # A user can have can_manage access to another user, this grants
+      # full access to all that user's stuff.  To implement that we
+      # need to include those other users in the permission query.
+      user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: ":user_uuids", perm_level: 1}
+
       # Note: it is possible to combine the direct_check and
       # owner_check into a single EXISTS() clause, however it turns
       # out query optimizer doesn't like it and forces a sequential
@@ -316,13 +331,28 @@ class ArvadosModel < ApplicationRecord
 
       # Match a direct read permission link from the user to the record uuid
       direct_check = "#{sql_table}.uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
-                     "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check})"
+                     "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check})"
 
-      # Match a read permission link from the user to the record's owner_uuid
+      # Match a read permission for the user to the record's
+      # owner_uuid.  This is so we can have a permissions table that
+      # mostly consists of users and groups (projects are a type of
+      # group) and not have to compute and list user permission to
+      # every single object in the system.
+      #
+      # Don't do this for API keys (special behavior) or groups
+      # (already covered by direct_check).
+      #
+      # The traverse_owned flag indicates whether the permission to
+      # read an object also implies transitive permission to read
+      # things the object owns.  The situation where this is important
+      # are determining if we can read an object owned by another
+      # user.  This makes it possible to have permission to read the
+      # user record without granting permission to read things the
+      # other user owns.
       owner_check = ""
       if sql_table != "api_client_authorizations" and sql_table != "groups" then
         owner_check = "OR #{sql_table}.owner_uuid IN (SELECT target_uuid FROM #{PERMISSION_VIEW} "+
-          "WHERE user_uuid IN (:user_uuids) AND perm_level >= 1 #{trashed_check} AND target_owner_uuid IS NOT NULL) "
+          "WHERE user_uuid IN (#{user_uuids_subquery}) AND perm_level >= 1 #{trashed_check} AND traverse_owned) "
       end
 
       links_cond = ""
@@ -331,7 +361,7 @@ class ArvadosModel < ApplicationRecord
         # users some permission _or_ gives anyone else permission to
         # view one of the authorized users.
         links_cond = "OR (#{sql_table}.link_class IN (:permission_link_classes) AND "+
-                       "(#{sql_table}.head_uuid IN (:user_uuids) OR #{sql_table}.tail_uuid IN (:user_uuids)))"
+                       "(#{sql_table}.head_uuid IN (#{user_uuids_subquery}) OR #{sql_table}.tail_uuid IN (#{user_uuids_subquery})))"
       end
 
       sql_conds = "(#{direct_check} #{owner_check} #{links_cond}) #{exclude_trashed_records}"
@@ -544,6 +574,9 @@ class ArvadosModel < ApplicationRecord
           logger.warn "User #{current_user.uuid} tried to set ownership of #{self.class.to_s} #{self.uuid} but does not have permission to write #{which} owner_uuid #{check_uuid}"
           errors.add :owner_uuid, "cannot be set or changed without write permission on #{which} owner"
           raise PermissionDeniedError
+        elsif rsc_class == Group && Group.find_by_uuid(owner_uuid).group_class != "project"
+          errors.add :owner_uuid, "must be a project"
+          raise PermissionDeniedError
         end
       end
     else
@@ -552,7 +585,7 @@ class ArvadosModel < ApplicationRecord
       # itself.
       if !current_user.can?(write: self.uuid)
         logger.warn "User #{current_user.uuid} tried to modify #{self.class.to_s} #{self.uuid} without write permission"
-        errors.add :uuid, "is not writable"
+        errors.add :uuid, " #{uuid} is not writable by #{current_user.uuid}"
         raise PermissionDeniedError
       end
     end
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 376be55ffb..912a801a6f 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -570,8 +570,13 @@ class Container < ArvadosModel
          return errors.add :auth_uuid, 'is readonly'
     end
     if not [Locked, Running].include? self.state
-      # don't need one
-      self.auth.andand.update_attributes(expires_at: db_current_time)
+      # Don't need one. If auth already exists, expire it.
+      #
+      # We use db_transaction_time here (not db_current_time) to
+      # ensure the token doesn't validate later in the same
+      # transaction (e.g., in a test case) by satisfying expires_at >
+      # transaction timestamp.
+      self.auth.andand.update_attributes(expires_at: db_transaction_time)
       self.auth = nil
       return
     elsif self.auth
diff --git a/services/api/app/models/database_seeds.rb b/services/api/app/models/database_seeds.rb
index 6e7ab9b076..39f491503e 100644
--- a/services/api/app/models/database_seeds.rb
+++ b/services/api/app/models/database_seeds.rb
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+require 'update_permissions'
+
 class DatabaseSeeds
   extend CurrentApiClient
   def self.install
@@ -12,5 +14,7 @@ class DatabaseSeeds
     anonymous_group_read_permission
     anonymous_user
     empty_collection
+    refresh_permissions
+    refresh_trashed
   end
 end
diff --git a/services/api/app/models/group.rb b/services/api/app/models/group.rb
index 1f2b0d8b77..02c6a242f9 100644
--- a/services/api/app/models/group.rb
+++ b/services/api/app/models/group.rb
@@ -17,9 +17,18 @@ class Group < ArvadosModel
   attribute :properties, :jsonbHash, default: {}
 
   validate :ensure_filesystem_compatible_name
-  after_create :invalidate_permissions_cache
-  after_update :maybe_invalidate_permissions_cache
+  validate :check_group_class
   before_create :assign_name
+  after_create :after_ownership_change
+  after_create :update_trash
+
+  before_update :before_ownership_change
+  after_update :after_ownership_change
+
+  after_create :add_role_manage_link
+
+  after_update :update_trash
+  before_destroy :clear_permissions_and_trash
 
   api_accessible :user, extend: :common do |t|
     t.add :name
@@ -38,18 +47,67 @@ class Group < ArvadosModel
     super if group_class == 'project'
   end
 
-  def maybe_invalidate_permissions_cache
-    if uuid_changed? or owner_uuid_changed? or is_trashed_changed?
-      # This can change users' permissions on other groups as well as
-      # this one.
-      invalidate_permissions_cache
+  def check_group_class
+    if group_class != 'project' && group_class != 'role'
+      errors.add :group_class, "value must be one of 'project' or 'role', was '#{group_class}'"
+    end
+    if group_class_changed? && !group_class_was.nil?
+      errors.add :group_class, "cannot be modified after record is created"
+    end
+  end
+
+  def update_trash
+    if trash_at_changed? or owner_uuid_changed?
+      # The group was added or removed from the trash.
+      #
+      # Strategy:
+      #   Compute project subtree, propagating trash_at to subprojects
+      #   Remove groups that don't belong from trash
+      #   Add/update groups that do belong in the trash
+
+      temptable = "group_subtree_#{rand(2**64).to_s(10)}"
+      ActiveRecord::Base.connection.exec_query %{
+create temporary table #{temptable} on commit drop
+as select * from project_subtree_with_trash_at($1, LEAST($2, $3)::timestamp)
+},
+                                               'Group.update_trash.select',
+                                               [[nil, self.uuid],
+                                                [nil, TrashedGroup.find_by_group_uuid(self.owner_uuid).andand.trash_at],
+                                                [nil, self.trash_at]]
+
+      ActiveRecord::Base.connection.exec_delete %{
+delete from trashed_groups where group_uuid in (select target_uuid from #{temptable} where trash_at is NULL);
+},
+                                            "Group.update_trash.delete"
+
+      ActiveRecord::Base.connection.exec_query %{
+insert into trashed_groups (group_uuid, trash_at)
+  select target_uuid as group_uuid, trash_at from #{temptable} where trash_at is not NULL
+on conflict (group_uuid) do update set trash_at=EXCLUDED.trash_at;
+},
+                                            "Group.update_trash.insert"
     end
   end
 
-  def invalidate_permissions_cache
-    # Ensure a new group can be accessed by the appropriate users
-    # immediately after being created.
-    User.invalidate_permissions_cache self.async_permissions_update
+  def before_ownership_change
+    if owner_uuid_changed? and !self.owner_uuid_was.nil?
+      MaterializedPermission.where(user_uuid: owner_uuid_was, target_uuid: uuid).delete_all
+      update_permissions self.owner_uuid_was, self.uuid, REVOKE_PERM
+    end
+  end
+
+  def after_ownership_change
+    if owner_uuid_changed?
+      update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+    end
+  end
+
+  def clear_permissions_and_trash
+    MaterializedPermission.where(target_uuid: uuid).delete_all
+    ActiveRecord::Base.connection.exec_delete %{
+delete from trashed_groups where group_uuid=$1
+}, "Group.clear_permissions_and_trash", [[nil, self.uuid]]
+
   end
 
   def assign_name
@@ -58,4 +116,33 @@ class Group < ArvadosModel
     end
     true
   end
+
+  def ensure_owner_uuid_is_permitted
+    if group_class == "role"
+      @requested_manager_uuid = nil
+      if new_record?
+        @requested_manager_uuid = owner_uuid
+        self.owner_uuid = system_user_uuid
+        return true
+      end
+      if self.owner_uuid != system_user_uuid
+        raise "Owner uuid for role must be system user"
+      end
+      raise PermissionDeniedError unless current_user.can?(manage: uuid)
+      true
+    else
+      super
+    end
+  end
+
+  def add_role_manage_link
+    if group_class == "role" && @requested_manager_uuid
+      act_as_system_user do
+       Link.create!(tail_uuid: @requested_manager_uuid,
+                    head_uuid: self.uuid,
+                    link_class: "permission",
+                    name: "can_manage")
+      end
+    end
+  end
 end
diff --git a/services/api/app/models/link.rb b/services/api/app/models/link.rb
index ad7800fe67..e4ba7f3de1 100644
--- a/services/api/app/models/link.rb
+++ b/services/api/app/models/link.rb
@@ -11,12 +11,13 @@ class Link < ArvadosModel
   # already know how to properly treat them.
   attribute :properties, :jsonbHash, default: {}
 
-  before_create :permission_to_attach_to_objects
-  before_update :permission_to_attach_to_objects
-  after_update :maybe_invalidate_permissions_cache
-  after_create :maybe_invalidate_permissions_cache
-  after_destroy :maybe_invalidate_permissions_cache
   validate :name_links_are_obsolete
+  validate :permission_to_attach_to_objects
+  before_update :restrict_alter_permissions
+  after_update :call_update_permissions
+  after_create :call_update_permissions
+  before_destroy :clear_permissions
+  after_destroy :check_permissions
 
   api_accessible :user, extend: :common do |t|
     t.add :tail_uuid
@@ -49,13 +50,37 @@ class Link < ArvadosModel
     # All users can write links that don't affect permissions
     return true if self.link_class != 'permission'
 
+    if PERM_LEVEL[self.name].nil?
+      errors.add(:name, "is invalid permission, must be one of 'can_read', 'can_write', 'can_manage', 'can_login'")
+      return false
+    end
+
+    rsc_class = ArvadosModel::resource_class_for_uuid tail_uuid
+    if rsc_class == Group
+      tail_obj = Group.find_by_uuid(tail_uuid)
+      if tail_obj.nil?
+        errors.add(:tail_uuid, "does not exist")
+        return false
+      end
+      if tail_obj.group_class != "role"
+        errors.add(:tail_uuid, "must be a user or role, was group with group_class #{tail_obj.group_class}")
+        return false
+      end
+    elsif rsc_class != User
+      errors.add(:tail_uuid, "must be a user or role")
+      return false
+    end
+
     # Administrators can grant permissions
     return true if current_user.is_admin
 
     head_obj = ArvadosModel.find_by_uuid(head_uuid)
 
     # No permission links can be pointed to past collection versions
-    return false if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
+    if head_obj.is_a?(Collection) && head_obj.current_version_uuid != head_uuid
+      errors.add(:head_uuid, "cannot point to a past version of a collection")
+      return false
+    end
 
     # All users can grant permissions on objects they own or can manage
     return true if current_user.can?(manage: head_obj)
@@ -64,15 +89,38 @@ class Link < ArvadosModel
     false
   end
 
-  def maybe_invalidate_permissions_cache
+  def restrict_alter_permissions
+    return true if self.link_class != 'permission' && self.link_class_was != 'permission'
+
+    return true if current_user.andand.uuid == system_user.uuid
+
+    if link_class_changed? || tail_uuid_changed? || head_uuid_changed?
+      raise "Can only alter permission link level"
+    end
+  end
+
+  PERM_LEVEL = {
+    'can_read' => 1,
+    'can_login' => 1,
+    'can_write' => 2,
+    'can_manage' => 3,
+  }
+
+  def call_update_permissions
+    if self.link_class == 'permission'
+      update_permissions tail_uuid, head_uuid, PERM_LEVEL[name], self.uuid
+    end
+  end
+
+  def clear_permissions
+    if self.link_class == 'permission'
+      update_permissions tail_uuid, head_uuid, REVOKE_PERM, self.uuid
+    end
+  end
+
+  def check_permissions
     if self.link_class == 'permission'
-      # Clearing the entire permissions cache can generate many
-      # unnecessary queries if many active users are not affected by
-      # this change. In such cases it would be better to search cached
-      # permissions for head_uuid and tail_uuid, and invalidate the
-      # cache for only those users. (This would require a browseable
-      # cache.)
-      User.invalidate_permissions_cache
+      check_permissions_against_full_refresh
     end
   end
 
diff --git a/services/api/app/models/materialized_permission.rb b/services/api/app/models/materialized_permission.rb
new file mode 100644
index 0000000000..24ba6737ae
--- /dev/null
+++ b/services/api/app/models/materialized_permission.rb
@@ -0,0 +1,6 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class MaterializedPermission < ApplicationRecord
+end
diff --git a/services/api/app/models/trashed_group.rb b/services/api/app/models/trashed_group.rb
new file mode 100644
index 0000000000..5c85946bae
--- /dev/null
+++ b/services/api/app/models/trashed_group.rb
@@ -0,0 +1,6 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class TrashedGroup < ApplicationRecord
+end
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index dd447ca51a..64facaa98e 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 require 'can_be_an_owner'
-require 'refresh_permission_view'
 
 class User < ArvadosModel
   include HasUuid
@@ -28,25 +27,31 @@ class User < ArvadosModel
     user.username.nil? and user.username_changed?
   }
   before_update :setup_on_activate
+
   before_create :check_auto_admin
   before_create :set_initial_username, :if => Proc.new { |user|
     user.username.nil? and user.email
   }
+  after_create :after_ownership_change
   after_create :setup_on_activate
   after_create :add_system_group_permission_link
-  after_create :invalidate_permissions_cache
   after_create :auto_setup_new_user, :if => Proc.new { |user|
     Rails.configuration.Users.AutoSetupNewUsers and
     (user.uuid != system_user_uuid) and
     (user.uuid != anonymous_user_uuid)
   }
   after_create :send_admin_notifications
+
+  before_update :before_ownership_change
+  after_update :after_ownership_change
   after_update :send_profile_created_notification
   after_update :sync_repository_names, :if => Proc.new { |user|
     (user.uuid != system_user_uuid) and
     user.username_changed? and
     (not user.username_was.nil?)
   }
+  before_destroy :clear_permissions
+  after_destroy :remove_self_from_permissions
 
   has_many :authorized_keys, :foreign_key => :authorized_user_uuid, :primary_key => :uuid
   has_many :repositories, foreign_key: :owner_uuid, primary_key: :uuid
@@ -77,6 +82,12 @@ class User < ArvadosModel
      {read: true, write: true},
      {read: true, write: true, manage: true}]
 
+  VAL_FOR_PERM =
+    {:read => 1,
+     :write => 2,
+     :manage => 3}
+
+
   def full_name
     "#{first_name} #{last_name}".strip
   end
@@ -88,7 +99,7 @@ class User < ArvadosModel
   end
 
   def groups_i_can(verb)
-    my_groups = self.group_permissions.select { |uuid, mask| mask[verb] }.keys
+    my_groups = self.group_permissions(VAL_FOR_PERM[verb]).keys
     if verb == :read
       my_groups << anonymous_group_uuid
     end
@@ -107,60 +118,68 @@ class User < ArvadosModel
         end
       end
       next if target_uuid == self.uuid
-      next if (group_permissions[target_uuid] and
-               group_permissions[target_uuid][action])
-      if target.respond_to? :owner_uuid
-        next if target.owner_uuid == self.uuid
-        next if (group_permissions[target.owner_uuid] and
-                 group_permissions[target.owner_uuid][action])
-      end
-      sufficient_perms = case action
-                         when :manage
-                           ['can_manage']
-                         when :write
-                           ['can_manage', 'can_write']
-                         when :read
-                           ['can_manage', 'can_write', 'can_read']
-                         else
-                           # (Skip this kind of permission opportunity
-                           # if action is an unknown permission type)
-                         end
-      if sufficient_perms
-        # Check permission links with head_uuid pointing directly at
-        # the target object. If target is a Group, this is redundant
-        # and will fail except [a] if permission caching is broken or
-        # [b] during a race condition, where a permission link has
-        # *just* been added.
-        if Link.where(link_class: 'permission',
-                      name: sufficient_perms,
-                      tail_uuid: groups_i_can(action) + [self.uuid],
-                      head_uuid: target_uuid).any?
-          next
-        end
+
+      target_owner_uuid = target.owner_uuid if target.respond_to? :owner_uuid
+
+      user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: "$3"}
+
+      unless ActiveRecord::Base.connection.
+        exec_query(%{
+SELECT 1 FROM #{PERMISSION_VIEW}
+  WHERE user_uuid in (#{user_uuids_subquery}) and
+        ((target_uuid = $2 and perm_level >= $3)
+         or (target_uuid = $4 and perm_level >= $3 and traverse_owned))
+},
+                  # "name" arg is a query label that appears in logs:
+                   "user_can_query",
+                   [[nil, self.uuid],
+                    [nil, target_uuid],
+                    [nil, VAL_FOR_PERM[action]],
+                    [nil, target_owner_uuid]]
+                  ).any?
+        return false
       end
-      return false
     end
     true
   end
 
-  def self.invalidate_permissions_cache(async=false)
-    refresh_permission_view(async)
+  def before_ownership_change
+    if owner_uuid_changed? and !self.owner_uuid_was.nil?
+      MaterializedPermission.where(user_uuid: owner_uuid_was, target_uuid: uuid).delete_all
+      update_permissions self.owner_uuid_was, self.uuid, REVOKE_PERM
+    end
   end
 
-  def invalidate_permissions_cache
-    User.invalidate_permissions_cache
+  def after_ownership_change
+    if owner_uuid_changed?
+      update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
+    end
+  end
+
+  def clear_permissions
+    MaterializedPermission.where("user_uuid = ? and target_uuid != ?", uuid, uuid).delete_all
+  end
+
+  def remove_self_from_permissions
+    MaterializedPermission.where("target_uuid = ?", uuid).delete_all
+    check_permissions_against_full_refresh
   end
 
   # Return a hash of {user_uuid: group_perms}
+  #
+  # note: this does not account for permissions that a user gains by
+  # having can_manage on another user.
   def self.all_group_permissions
     all_perms = {}
     ActiveRecord::Base.connection.
-      exec_query("SELECT user_uuid, target_owner_uuid, perm_level, trashed
+      exec_query(%{
+SELECT user_uuid, target_uuid, perm_level
                   FROM #{PERMISSION_VIEW}
-                  WHERE target_owner_uuid IS NOT NULL",
+                  WHERE traverse_owned
+},
                   # "name" arg is a query label that appears in logs:
-                  "all_group_permissions",
-                  ).rows.each do |user_uuid, group_uuid, max_p_val, trashed|
+                 "all_group_permissions").
+      rows.each do |user_uuid, group_uuid, max_p_val|
       all_perms[user_uuid] ||= {}
       all_perms[user_uuid][group_uuid] = PERMS_FOR_VAL[max_p_val.to_i]
     end
@@ -170,18 +189,23 @@ class User < ArvadosModel
   # Return a hash of {group_uuid: perm_hash} where perm_hash[:read]
   # and perm_hash[:write] are true if this user can read and write
   # objects owned by group_uuid.
-  def group_permissions
-    group_perms = {self.uuid => {:read => true, :write => true, :manage => true}}
+  def group_permissions(level=1)
+    group_perms = {}
+
+    user_uuids_subquery = USER_UUIDS_SUBQUERY_TEMPLATE % {user: "$1", perm_level: "$2"}
+
     ActiveRecord::Base.connection.
-      exec_query("SELECT target_owner_uuid, perm_level, trashed
-                  FROM #{PERMISSION_VIEW}
-                  WHERE user_uuid = $1
-                  AND target_owner_uuid IS NOT NULL",
+      exec_query(%{
+SELECT target_uuid, perm_level
+  FROM #{PERMISSION_VIEW}
+  WHERE user_uuid in (#{user_uuids_subquery}) and perm_level >= $2
+},
                   # "name" arg is a query label that appears in logs:
-                  "group_permissions for #{uuid}",
+                  "User.group_permissions",
                   # "binds" arg is an array of [col_id, value] for '$1' vars:
-                  [[nil, uuid]],
-                ).rows.each do |group_uuid, max_p_val, trashed|
+                  [[nil, uuid],
+                   [nil, level]]).
+      rows.each do |group_uuid, max_p_val|
       group_perms[group_uuid] = PERMS_FOR_VAL[max_p_val.to_i]
     end
     group_perms
@@ -238,8 +262,14 @@ class User < ArvadosModel
   end
 
   def must_unsetup_to_deactivate
-    if self.is_active_changed? &&
-       self.is_active_was == true &&
+    if !self.new_record? &&
+       self.uuid[0..4] == Rails.configuration.Login.LoginCluster &&
+       self.uuid[0..4] != Rails.configuration.ClusterID
+      # OK to update our local record to whatever the LoginCluster
+      # reports, because self-activate is not allowed.
+      return
+    elsif self.is_active_changed? &&
+       self.is_active_was &&
        !self.is_active
 
       group = Group.where(name: 'All users').select do |g|
@@ -303,6 +333,18 @@ class User < ArvadosModel
       self.uuid = new_uuid
       save!(validate: false)
       change_all_uuid_refs(old_uuid: old_uuid, new_uuid: new_uuid)
+    ActiveRecord::Base.connection.exec_update %{
+update #{PERMISSION_VIEW} set user_uuid=$1 where user_uuid = $2
+},
+                                             'User.update_uuid.update_permissions_user_uuid',
+                                             [[nil, new_uuid],
+                                              [nil, old_uuid]]
+      ActiveRecord::Base.connection.exec_update %{
+update #{PERMISSION_VIEW} set target_uuid=$1 where target_uuid = $2
+},
+                                            'User.update_uuid.update_permissions_target_uuid',
+                                             [[nil, new_uuid],
+                                              [nil, old_uuid]]
     end
   end
 
@@ -328,6 +370,9 @@ class User < ArvadosModel
       raise "user does not exist" if !new_user
       raise "cannot merge to an already merged user" if new_user.redirect_to_user_uuid
 
+      self.clear_permissions
+      new_user.clear_permissions
+
       # If 'self' is a remote user, don't transfer authorizations
       # (i.e. ability to access the account) to the new user, because
       # that gives the remote site the ability to access the 'new'
@@ -402,7 +447,12 @@ class User < ArvadosModel
       if redirect_to_new_user
         update_attributes!(redirect_to_user_uuid: new_user.uuid, username: nil)
       end
-      invalidate_permissions_cache
+      skip_check_permissions_against_full_refresh do
+        update_permissions self.uuid, self.uuid, CAN_MANAGE_PERM
+        update_permissions new_user.uuid, new_user.uuid, CAN_MANAGE_PERM
+        update_permissions new_user.owner_uuid, new_user.uuid, CAN_MANAGE_PERM
+      end
+      update_permissions self.owner_uuid, self.uuid, CAN_MANAGE_PERM
     end
   end
 
diff --git a/services/api/config/arvados_config.rb b/services/api/config/arvados_config.rb
index 502e3e787d..f63f8af033 100644
--- a/services/api/config/arvados_config.rb
+++ b/services/api/config/arvados_config.rb
@@ -37,8 +37,8 @@ EOS
   # Real values will be copied from globals by omniauth_init.rb. For
   # now, assign some strings so the generic *.yml config loader
   # doesn't overwrite them or complain that they're missing.
-  Rails.configuration.Login["ProviderAppID"] = 'xxx'
-  Rails.configuration.Login["ProviderAppSecret"] = 'xxx'
+  Rails.configuration.Login["SSO"]["ProviderAppID"] = 'xxx'
+  Rails.configuration.Login["SSO"]["ProviderAppSecret"] = 'xxx'
   Rails.configuration.Services["SSO"]["ExternalURL"] = '//xxx'
   WARNED_OMNIAUTH_CONFIG = true
 end
@@ -106,8 +106,8 @@ arvcfg.declare_config "Users.EmailSubjectPrefix", String, :email_subject_prefix
 arvcfg.declare_config "Users.UserNotifierEmailFrom", String, :user_notifier_email_from
 arvcfg.declare_config "Users.NewUserNotificationRecipients", Hash, :new_user_notification_recipients, ->(cfg, k, v) { arrayToHash cfg, "Users.NewUserNotificationRecipients", v }
 arvcfg.declare_config "Users.NewInactiveUserNotificationRecipients", Hash, :new_inactive_user_notification_recipients, method(:arrayToHash)
-arvcfg.declare_config "Login.ProviderAppSecret", String, :sso_app_secret
-arvcfg.declare_config "Login.ProviderAppID", String, :sso_app_id
+arvcfg.declare_config "Login.SSO.ProviderAppSecret", String, :sso_app_secret
+arvcfg.declare_config "Login.SSO.ProviderAppID", String, :sso_app_id
 arvcfg.declare_config "Login.LoginCluster", String
 arvcfg.declare_config "Login.RemoteTokenRefresh", ActiveSupport::Duration
 arvcfg.declare_config "TLS.Insecure", Boolean, :sso_insecure
@@ -190,6 +190,7 @@ dbcfg.declare_config "PostgreSQL.Connection.password", String, :password
 dbcfg.declare_config "PostgreSQL.Connection.dbname", String, :database
 dbcfg.declare_config "PostgreSQL.Connection.template", String, :template
 dbcfg.declare_config "PostgreSQL.Connection.encoding", String, :encoding
+dbcfg.declare_config "PostgreSQL.Connection.collation", String, :collation
 
 application_config = {}
 %w(application.default application).each do |cfgfile|
@@ -257,6 +258,8 @@ if ::Rails.env.to_s == "test"
   # Use template0 when creating a new database. Avoids
   # character-encoding/collation problems.
   $arvados_config["PostgreSQL"]["Connection"]["template"] = "template0"
+  # Some test cases depend on en_US.UTF-8 collation.
+  $arvados_config["PostgreSQL"]["Connection"]["collation"] = "en_US.UTF-8"
 end
 
 if $arvados_config["PostgreSQL"]["Connection"]["password"].empty?
@@ -279,6 +282,7 @@ ENV["DATABASE_URL"] = "postgresql://#{$arvados_config["PostgreSQL"]["Connection"
                       "#{dbhost}/#{$arvados_config["PostgreSQL"]["Connection"]["dbname"]}?"+
                       "template=#{$arvados_config["PostgreSQL"]["Connection"]["template"]}&"+
                       "encoding=#{$arvados_config["PostgreSQL"]["Connection"]["client_encoding"]}&"+
+                      "collation=#{$arvados_config["PostgreSQL"]["Connection"]["collation"]}&"+
                       "pool=#{$arvados_config["PostgreSQL"]["ConnectionPool"]}"
 
 Server::Application.configure do
diff --git a/services/api/config/initializers/omniauth_init.rb b/services/api/config/initializers/omniauth_init.rb
index 5610999a94..a1b2356bd5 100644
--- a/services/api/config/initializers/omniauth_init.rb
+++ b/services/api/config/initializers/omniauth_init.rb
@@ -9,14 +9,14 @@
 
 if defined? CUSTOM_PROVIDER_URL
   Rails.logger.warn "Copying omniauth from globals in legacy config file."
-  Rails.configuration.Login["ProviderAppID"] = APP_ID
-  Rails.configuration.Login["ProviderAppSecret"] = APP_SECRET
-  Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL
+  Rails.configuration.Login["SSO"]["ProviderAppID"] = APP_ID
+  Rails.configuration.Login["SSO"]["ProviderAppSecret"] = APP_SECRET
+  Rails.configuration.Services["SSO"]["ExternalURL"] = CUSTOM_PROVIDER_URL.sub(/\/$/, "") + "/"
 else
   Rails.application.config.middleware.use OmniAuth::Builder do
     provider(:josh_id,
-             Rails.configuration.Login["ProviderAppID"],
-             Rails.configuration.Login["ProviderAppSecret"],
+             Rails.configuration.Login["SSO"]["ProviderAppID"],
+             Rails.configuration.Login["SSO"]["ProviderAppSecret"],
              Rails.configuration.Services["SSO"]["ExternalURL"])
   end
   OmniAuth.config.on_failure = StaticController.action(:login_failure)
diff --git a/services/api/config/initializers/time_zone.rb b/services/api/config/initializers/time_zone.rb
new file mode 100644
index 0000000000..cedd8f3e4a
--- /dev/null
+++ b/services/api/config/initializers/time_zone.rb
@@ -0,0 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+ActiveRecord::Base.connection.class.set_callback :checkout, :after do
+  # If the database connection is in a time zone other than UTC,
+  # "timestamp" values don't behave as desired.
+  #
+  # For example, ['select now() > ?', Time.now] returns true in time
+  # zones +0100 and UTC (which makes sense since Time.now is evaluated
+  # before now()), but false in time zone -0100 (now() returns an
+  # earlier clock time, and its time zone is dropped when comparing to
+  # a "timestamp without time zone").
+  raw_connection.sync_exec("SET TIME ZONE 'UTC'")
+end
diff --git a/services/api/db/migrate/20200501150153_permission_table.rb b/services/api/db/migrate/20200501150153_permission_table.rb
new file mode 100644
index 0000000000..4f9ea156dc
--- /dev/null
+++ b/services/api/db/migrate/20200501150153_permission_table.rb
@@ -0,0 +1,362 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require '20200501150153_permission_table_constants'
+
+class PermissionTable < ActiveRecord::Migration[5.0]
+  def up
+    # This is a major migration.  We are replacing the
+    # materialized_permission_view, which is fully recomputed any time
+    # a permission changes (and becomes very expensive as the number
+    # of users/groups becomes large), with a new strategy that only
+    # recomputes permissions for the subset of objects that are
+    # potentially affected by the addition or removal of a permission
+    # relationship (i.e. ownership or a permission link).
+    #
+    # This also disentangles the concept of "trashed groups" from the
+    # permissions system.  Updating trashed items follows a similar
+    # (but less complicated) strategy to updating permissions, so it
+    # may be helpful to look at that first.
+
+    ActiveRecord::Base.connection.execute "DROP MATERIALIZED VIEW IF EXISTS materialized_permission_view;"
+    drop_table :permission_refresh_lock
+
+    # This table stores the set of trashed groups and their trash_at
+    # time.  Used to exclude trashed projects and their contents when
+    # getting object listings.
+    create_table :trashed_groups, :id => false do |t|
+      t.string :group_uuid
+      t.datetime :trash_at
+    end
+    add_index :trashed_groups, :group_uuid, :unique => true
+
+    ActiveRecord::Base.connection.execute %{
+create or replace function project_subtree_with_trash_at (starting_uuid varchar(27), starting_trash_at timestamp)
+returns table (target_uuid varchar(27), trash_at timestamp)
+STABLE
+language SQL
+as $$
+/* Starting from a project, recursively traverse all the projects
+  underneath it and return a set of project uuids and trash_at times
+  (may be null).  The initial trash_at can be a timestamp or null.
+  The trash_at time propagates downward to groups it owns, i.e. when a
+  group is trashed, everything underneath it in the ownership
+  hierarchy is also considered trashed.  However, this is fact is
+  recorded in the trashed_groups table, not by updating trash_at field
+  in the groups table.
+*/
+WITH RECURSIVE
+        project_subtree(uuid, trash_at) as (
+        values (starting_uuid, starting_trash_at)
+        union
+        select groups.uuid, LEAST(project_subtree.trash_at, groups.trash_at)
+          from groups join project_subtree on (groups.owner_uuid = project_subtree.uuid)
+        )
+        select uuid, trash_at from project_subtree;
+$$;
+}
+
+    # Now populate the table.  For a non-test databse this is the only
+    # time this ever happens, after this the trash table is updated
+    # incrementally.  See app/models/group.rb#update_trash
+    refresh_trashed
+
+    # The table to store the flattened permissions.  This is almost
+    # exactly the same as the old materalized_permission_view except
+    # that the target_owner_uuid colunm in the view is now just a
+    # boolean traverse_owned (the column was only ever tested for null
+    # or non-null).
+    #
+    # For details on how this table is used to apply permissions to
+    # queries, see app/models/arvados_model.rb#readable_by
+    #
+    create_table :materialized_permissions, :id => false do |t|
+      t.string :user_uuid
+      t.string :target_uuid
+      t.integer :perm_level
+      t.boolean :traverse_owned
+    end
+    add_index :materialized_permissions, [:user_uuid, :target_uuid], unique: true, name: 'permission_user_target'
+    add_index :materialized_permissions, [:target_uuid], unique: false, name: 'permission_target'
+
+    ActiveRecord::Base.connection.execute %{
+create or replace function should_traverse_owned (starting_uuid varchar(27),
+                                                  starting_perm integer)
+  returns bool
+IMMUTABLE
+language SQL
+as $$
+/* Helper function.  Determines if permission on an object implies
+   transitive permission to things the object owns.  This is always
+   true for groups, but only true for users when the permission level
+   is can_manage.
+*/
+select starting_uuid like '_____-j7d0g-_______________' or
+       (starting_uuid like '_____-tpzed-_______________' and starting_perm >= 3);
+$$;
+}
+
+    # Merge all permission relationships into a single view.  This
+    # consists of: groups owned by users and projects, users owned
+    # by other users, users have permission on themselves,
+    # and explicit permission links.
+    #
+    # A SQL view gets inlined into the query where it is used as a
+    # subquery.  This enables the query planner to inject constraints,
+    # so it only has to look up edges it plans to traverse and avoid a brute
+    # force query of all edges.
+    ActiveRecord::Base.connection.execute %{
+create view permission_graph_edges as
+  select groups.owner_uuid as tail_uuid, groups.uuid as head_uuid,
+         (3) as val, groups.uuid as edge_id from groups
+union all
+  select users.owner_uuid as tail_uuid, users.uuid as head_uuid,
+         (3) as val, users.uuid as edge_id from users
+union all
+  select users.uuid as tail_uuid, users.uuid as head_uuid,
+         (3) as val, '' as edge_id from users
+union all
+  select links.tail_uuid,
+         links.head_uuid,
+         CASE
+           WHEN links.name = 'can_read'   THEN 1
+           WHEN links.name = 'can_login'  THEN 1
+           WHEN links.name = 'can_write'  THEN 2
+           WHEN links.name = 'can_manage' THEN 3
+           ELSE 0
+         END as val,
+         links.uuid as edge_id
+      from links
+      where links.link_class='permission'
+}
+
+    # This is used to ensure that the permission edge passed into
+    # compute_permission_subgraph takes replaces the existing edge in
+    # the "edges" view that is about to be removed.
+    edge_perm = %{
+case (edges.edge_id = perm_edge_id)
+                               when true then starting_perm
+                               else edges.val
+                            end
+}
+
+    # The primary function to compute permissions for a subgraph.
+    # Comments on how it works are inline.
+    #
+    # Due to performance issues due to the query optimizer not
+    # working across function and "with" expression boundaries, I
+    # had to fall back on using string templates for repeated code
+    # in order to inline it.
+
+    ActiveRecord::Base.connection.execute %{
+create or replace function compute_permission_subgraph (perm_origin_uuid varchar(27),
+                                                        starting_uuid varchar(27),
+                                                        starting_perm integer,
+                                                        perm_edge_id varchar(27))
+returns table (user_uuid varchar(27), target_uuid varchar(27), val integer, traverse_owned bool)
+STABLE
+language SQL
+as $$
+
+/* The purpose of this function is to compute the permissions for a
+   subgraph of the database, starting from a given edge.  The newly
+   computed permissions are used to add and remove rows from the main
+   permissions table.
+
+   perm_origin_uuid: The object that 'gets' the permission.
+
+   starting_uuid: The starting object the permission applies to.
+
+   starting_perm: The permission that perm_origin_uuid 'has' on
+                  starting_uuid One of 1, 2, 3 for can_read,
+                  can_write, can_manage respectively, or 0 to revoke
+                  permissions.
+
+   perm_edge_id: Identifies the permission edge that is being updated.
+                 Changes of ownership, this is starting_uuid.
+                 For links, this is the uuid of the link object.
+                 This is used to override the edge value in the database
+                 with starting_perm.  This is necessary when revoking
+                 permissions because the update happens before edge is
+                 actually removed.
+*/
+with
+  /* Starting from starting_uuid, determine the set of objects that
+     could be affected by this permission change.
+
+     Note: We don't traverse users unless it is an "identity"
+     permission (permission origin is self).
+  */
+  perm_from_start(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+    #{PERM_QUERY_TEMPLATE % {:base_case => %{
+             values (perm_origin_uuid, starting_uuid, starting_perm,
+                    should_traverse_owned(starting_uuid, starting_perm),
+                    (perm_origin_uuid = starting_uuid or starting_uuid not like '_____-tpzed-_______________'))
+},
+:edge_perm => edge_perm
+} }),
+
+  /* Find other inbound edges that grant permissions to 'targets' in
+     perm_from_start, and compute permissions that originate from
+     those.
+
+     This is necessary for two reasons:
+
+       1) Other users may have access to a subset of the objects
+       through other permission links than the one we started from.
+       If we don't recompute them, their permission will get dropped.
+
+       2) There may be more than one path through which a user gets
+       permission to an object.  For example, a user owns a project
+       and also shares it can_read with a group the user belongs
+       to. adding the can_read link must not overwrite the existing
+       can_manage permission granted by ownership.
+  */
+  additional_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+    #{PERM_QUERY_TEMPLATE % {:base_case => %{
+    select edges.tail_uuid as origin_uuid, edges.head_uuid as target_uuid, edges.val,
+           should_traverse_owned(edges.head_uuid, edges.val),
+           edges.head_uuid like '_____-j7d0g-_______________'
+      from permission_graph_edges as edges
+      where edges.edge_id != perm_edge_id and
+            edges.tail_uuid not in (select target_uuid from perm_from_start where target_uuid like '_____-j7d0g-_______________') and
+            edges.head_uuid in (select target_uuid from perm_from_start)
+},
+:edge_perm => edge_perm
+} }),
+
+  /* Combine the permissions computed in the first two phases. */
+  all_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+      select * from perm_from_start
+    union all
+      select * from additional_perms
+  )
+
+  /* The actual query that produces rows to be added or removed
+     from the materialized_permissions table.  This is the clever
+     bit.
+
+     Key insights:
+
+     * For every group, the materialized_permissions lists all users
+       that can access to that group.
+
+     * The all_perms subquery has computed permissions on on a set of
+       objects for all inbound "origins", which are users or groups.
+
+     * Permissions through groups are transitive.
+
+     We can infer:
+
+     1) The materialized_permissions table declares that user X has permission N on group Y
+     2) The all_perms result has determined group Y has permission M on object Z
+     3) Therefore, user X has permission min(N, M) on object Z
+
+     This allows us to efficiently determine the set of users that
+     have permissions on the subset of objects, without having to
+     follow the chain of permission back up to find those users.
+
+     In addition, because users always have permission on themselves, this
+     query also makes sure those permission rows are always
+     returned.
+  */
+  select v.user_uuid, v.target_uuid, max(v.perm_level), bool_or(v.traverse_owned) from
+    (select m.user_uuid,
+         u.target_uuid,
+         least(u.val, m.perm_level) as perm_level,
+         u.traverse_owned
+      from all_perms as u, materialized_permissions as m
+           where u.perm_origin_uuid = m.target_uuid AND m.traverse_owned
+           AND (m.user_uuid = m.target_uuid or m.target_uuid not like '_____-tpzed-_______________')
+    union all
+      select target_uuid as user_uuid, target_uuid, 3, true
+        from all_perms
+        where all_perms.target_uuid like '_____-tpzed-_______________') as v
+    group by v.user_uuid, v.target_uuid
+$$;
+     }
+
+    #
+    # Populate materialized_permissions by traversing permissions
+    # starting at each user.
+    #
+    refresh_permissions
+  end
+
+  def down
+    drop_table :materialized_permissions
+    drop_table :trashed_groups
+
+    ActiveRecord::Base.connection.execute "DROP function project_subtree_with_trash_at (varchar, timestamp);"
+    ActiveRecord::Base.connection.execute "DROP function compute_permission_subgraph (varchar, varchar, integer, varchar);"
+    ActiveRecord::Base.connection.execute "DROP function should_traverse_owned(varchar, integer);"
+    ActiveRecord::Base.connection.execute "DROP view permission_graph_edges;"
+
+    ActiveRecord::Base.connection.execute(%{
+CREATE MATERIALIZED VIEW materialized_permission_view AS
+ WITH RECURSIVE perm_value(name, val) AS (
+         VALUES ('can_read'::text,(1)::smallint), ('can_login'::text,1), ('can_write'::text,2), ('can_manage'::text,3)
+        ), perm_edges(tail_uuid, head_uuid, val, follow, trashed) AS (
+         SELECT links.tail_uuid,
+            links.head_uuid,
+            pv.val,
+            ((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
+            (0)::smallint AS trashed,
+            (0)::smallint AS followtrash
+           FROM ((public.links
+             LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
+             LEFT JOIN public.groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
+          WHERE ((links.link_class)::text = 'permission'::text)
+        UNION ALL
+         SELECT groups.owner_uuid,
+            groups.uuid,
+            3,
+            true AS bool,
+                CASE
+                    WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
+                    ELSE 0
+                END AS "case",
+            1
+           FROM public.groups
+        ), perm(val, follow, user_uuid, target_uuid, trashed) AS (
+         SELECT (3)::smallint AS val,
+            true AS follow,
+            (users.uuid)::character varying(32) AS user_uuid,
+            (users.uuid)::character varying(32) AS target_uuid,
+            (0)::smallint AS trashed
+           FROM public.users
+        UNION
+         SELECT (LEAST((perm_1.val)::integer, edges.val))::smallint AS val,
+            edges.follow,
+            perm_1.user_uuid,
+            (edges.head_uuid)::character varying(32) AS target_uuid,
+            ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
+           FROM (perm perm_1
+             JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
+        )
+ SELECT perm.user_uuid,
+    perm.target_uuid,
+    max(perm.val) AS perm_level,
+        CASE perm.follow
+            WHEN true THEN perm.target_uuid
+            ELSE NULL::character varying
+        END AS target_owner_uuid,
+    max(perm.trashed) AS trashed
+   FROM perm
+  GROUP BY perm.user_uuid, perm.target_uuid,
+        CASE perm.follow
+            WHEN true THEN perm.target_uuid
+            ELSE NULL::character varying
+        END
+  WITH NO DATA;
+}
+    )
+
+    add_index :materialized_permission_view, [:trashed, :target_uuid], name: 'permission_target_trashed'
+    add_index :materialized_permission_view, [:user_uuid, :trashed, :perm_level], name: 'permission_target_user_trashed_level'
+    create_table :permission_refresh_lock
+
+    ActiveRecord::Base.connection.execute 'REFRESH MATERIALIZED VIEW materialized_permission_view;'
+  end
+end
diff --git a/services/api/db/migrate/20200602141328_fix_roles_projects.rb b/services/api/db/migrate/20200602141328_fix_roles_projects.rb
new file mode 100644
index 0000000000..e17ef6de53
--- /dev/null
+++ b/services/api/db/migrate/20200602141328_fix_roles_projects.rb
@@ -0,0 +1,17 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'fix_roles_projects'
+
+class FixRolesProjects < ActiveRecord::Migration[5.0]
+  def up
+    # defined in a function for easy testing.
+    fix_roles_projects
+  end
+
+  def down
+    # This migration is not reversible.  However, the results are
+    # backwards compatible.
+  end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 88cd0baa2f..83987d0518 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -38,6 +38,216 @@ CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
 -- COMMENT ON EXTENSION pg_trgm IS 'text similarity measurement and index searching based on trigrams';
 
 
+--
+-- Name: compute_permission_subgraph(character varying, character varying, integer, character varying); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.compute_permission_subgraph(perm_origin_uuid character varying, starting_uuid character varying, starting_perm integer, perm_edge_id character varying) RETURNS TABLE(user_uuid character varying, target_uuid character varying, val integer, traverse_owned boolean)
+    LANGUAGE sql STABLE
+    AS $$
+
+/* The purpose of this function is to compute the permissions for a
+   subgraph of the database, starting from a given edge.  The newly
+   computed permissions are used to add and remove rows from the main
+   permissions table.
+
+   perm_origin_uuid: The object that 'gets' the permission.
+
+   starting_uuid: The starting object the permission applies to.
+
+   starting_perm: The permission that perm_origin_uuid 'has' on
+                  starting_uuid One of 1, 2, 3 for can_read,
+                  can_write, can_manage respectively, or 0 to revoke
+                  permissions.
+
+   perm_edge_id: Identifies the permission edge that is being updated.
+                 Changes of ownership, this is starting_uuid.
+                 For links, this is the uuid of the link object.
+                 This is used to override the edge value in the database
+                 with starting_perm.  This is necessary when revoking
+                 permissions because the update happens before edge is
+                 actually removed.
+*/
+with
+  /* Starting from starting_uuid, determine the set of objects that
+     could be affected by this permission change.
+
+     Note: We don't traverse users unless it is an "identity"
+     permission (permission origin is self).
+  */
+  perm_from_start(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+    
+WITH RECURSIVE
+        traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
+            
+             values (perm_origin_uuid, starting_uuid, starting_perm,
+                    should_traverse_owned(starting_uuid, starting_perm),
+                    (perm_origin_uuid = starting_uuid or starting_uuid not like '_____-tpzed-_______________'))
+
+          union
+            (select traverse_graph.origin_uuid,
+                    edges.head_uuid,
+                      least(
+case (edges.edge_id = perm_edge_id)
+                               when true then starting_perm
+                               else edges.val
+                            end
+,
+                            traverse_graph.val),
+                    should_traverse_owned(edges.head_uuid, edges.val),
+                    false
+             from permission_graph_edges as edges, traverse_graph
+             where traverse_graph.target_uuid = edges.tail_uuid
+             and (edges.tail_uuid like '_____-j7d0g-_______________' or
+                  traverse_graph.starting_set)))
+        select traverse_graph.origin_uuid, target_uuid, max(val) as val, bool_or(traverse_owned) as traverse_owned from traverse_graph
+        group by (traverse_graph.origin_uuid, target_uuid)
+),
+
+  /* Find other inbound edges that grant permissions to 'targets' in
+     perm_from_start, and compute permissions that originate from
+     those.
+
+     This is necessary for two reasons:
+
+       1) Other users may have access to a subset of the objects
+       through other permission links than the one we started from.
+       If we don't recompute them, their permission will get dropped.
+
+       2) There may be more than one path through which a user gets
+       permission to an object.  For example, a user owns a project
+       and also shares it can_read with a group the user belongs
+       to. adding the can_read link must not overwrite the existing
+       can_manage permission granted by ownership.
+  */
+  additional_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+    
+WITH RECURSIVE
+        traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
+            
+    select edges.tail_uuid as origin_uuid, edges.head_uuid as target_uuid, edges.val,
+           should_traverse_owned(edges.head_uuid, edges.val),
+           edges.head_uuid like '_____-j7d0g-_______________'
+      from permission_graph_edges as edges
+      where edges.edge_id != perm_edge_id and
+            edges.tail_uuid not in (select target_uuid from perm_from_start where target_uuid like '_____-j7d0g-_______________') and
+            edges.head_uuid in (select target_uuid from perm_from_start)
+
+          union
+            (select traverse_graph.origin_uuid,
+                    edges.head_uuid,
+                      least(
+case (edges.edge_id = perm_edge_id)
+                               when true then starting_perm
+                               else edges.val
+                            end
+,
+                            traverse_graph.val),
+                    should_traverse_owned(edges.head_uuid, edges.val),
+                    false
+             from permission_graph_edges as edges, traverse_graph
+             where traverse_graph.target_uuid = edges.tail_uuid
+             and (edges.tail_uuid like '_____-j7d0g-_______________' or
+                  traverse_graph.starting_set)))
+        select traverse_graph.origin_uuid, target_uuid, max(val) as val, bool_or(traverse_owned) as traverse_owned from traverse_graph
+        group by (traverse_graph.origin_uuid, target_uuid)
+),
+
+  /* Combine the permissions computed in the first two phases. */
+  all_perms(perm_origin_uuid, target_uuid, val, traverse_owned) as (
+      select * from perm_from_start
+    union all
+      select * from additional_perms
+  )
+
+  /* The actual query that produces rows to be added or removed
+     from the materialized_permissions table.  This is the clever
+     bit.
+
+     Key insights:
+
+     * For every group, the materialized_permissions lists all users
+       that can access to that group.
+
+     * The all_perms subquery has computed permissions on on a set of
+       objects for all inbound "origins", which are users or groups.
+
+     * Permissions through groups are transitive.
+
+     We can infer:
+
+     1) The materialized_permissions table declares that user X has permission N on group Y
+     2) The all_perms result has determined group Y has permission M on object Z
+     3) Therefore, user X has permission min(N, M) on object Z
+
+     This allows us to efficiently determine the set of users that
+     have permissions on the subset of objects, without having to
+     follow the chain of permission back up to find those users.
+
+     In addition, because users always have permission on themselves, this
+     query also makes sure those permission rows are always
+     returned.
+  */
+  select v.user_uuid, v.target_uuid, max(v.perm_level), bool_or(v.traverse_owned) from
+    (select m.user_uuid,
+         u.target_uuid,
+         least(u.val, m.perm_level) as perm_level,
+         u.traverse_owned
+      from all_perms as u, materialized_permissions as m
+           where u.perm_origin_uuid = m.target_uuid AND m.traverse_owned
+           AND (m.user_uuid = m.target_uuid or m.target_uuid not like '_____-tpzed-_______________')
+    union all
+      select target_uuid as user_uuid, target_uuid, 3, true
+        from all_perms
+        where all_perms.target_uuid like '_____-tpzed-_______________') as v
+    group by v.user_uuid, v.target_uuid
+$$;
+
+
+--
+-- Name: project_subtree_with_trash_at(character varying, timestamp without time zone); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.project_subtree_with_trash_at(starting_uuid character varying, starting_trash_at timestamp without time zone) RETURNS TABLE(target_uuid character varying, trash_at timestamp without time zone)
+    LANGUAGE sql STABLE
+    AS $$
+/* Starting from a project, recursively traverse all the projects
+  underneath it and return a set of project uuids and trash_at times
+  (may be null).  The initial trash_at can be a timestamp or null.
+  The trash_at time propagates downward to groups it owns, i.e. when a
+  group is trashed, everything underneath it in the ownership
+  hierarchy is also considered trashed.  However, this is fact is
+  recorded in the trashed_groups table, not by updating trash_at field
+  in the groups table.
+*/
+WITH RECURSIVE
+        project_subtree(uuid, trash_at) as (
+        values (starting_uuid, starting_trash_at)
+        union
+        select groups.uuid, LEAST(project_subtree.trash_at, groups.trash_at)
+          from groups join project_subtree on (groups.owner_uuid = project_subtree.uuid)
+        )
+        select uuid, trash_at from project_subtree;
+$$;
+
+
+--
+-- Name: should_traverse_owned(character varying, integer); Type: FUNCTION; Schema: public; Owner: -
+--
+
+CREATE FUNCTION public.should_traverse_owned(starting_uuid character varying, starting_perm integer) RETURNS boolean
+    LANGUAGE sql IMMUTABLE
+    AS $$
+/* Helper function.  Determines if permission on an object implies
+   transitive permission to things the object owns.  This is always
+   true for groups, but only true for users when the permission level
+   is can_manage.
+*/
+select starting_uuid like '_____-j7d0g-_______________' or
+       (starting_uuid like '_____-tpzed-_______________' and starting_perm >= 3);
+$$;
+
+
 SET default_tablespace = '';
 
 SET default_with_oids = false;
@@ -719,93 +929,17 @@ ALTER SEQUENCE public.logs_id_seq OWNED BY public.logs.id;
 
 
 --
--- Name: users; Type: TABLE; Schema: public; Owner: -
+-- Name: materialized_permissions; Type: TABLE; Schema: public; Owner: -
 --
 
-CREATE TABLE public.users (
-    id integer NOT NULL,
-    uuid character varying(255),
-    owner_uuid character varying(255) NOT NULL,
-    created_at timestamp without time zone NOT NULL,
-    modified_by_client_uuid character varying(255),
-    modified_by_user_uuid character varying(255),
-    modified_at timestamp without time zone,
-    email character varying(255),
-    first_name character varying(255),
-    last_name character varying(255),
-    identity_url character varying(255),
-    is_admin boolean,
-    prefs text,
-    updated_at timestamp without time zone NOT NULL,
-    default_owner_uuid character varying(255),
-    is_active boolean DEFAULT false,
-    username character varying(255),
-    redirect_to_user_uuid character varying
+CREATE TABLE public.materialized_permissions (
+    user_uuid character varying,
+    target_uuid character varying,
+    perm_level integer,
+    traverse_owned boolean
 );
 
 
---
--- Name: materialized_permission_view; Type: MATERIALIZED VIEW; Schema: public; Owner: -
---
-
-CREATE MATERIALIZED VIEW public.materialized_permission_view AS
- WITH RECURSIVE perm_value(name, val) AS (
-         VALUES ('can_read'::text,(1)::smallint), ('can_login'::text,1), ('can_write'::text,2), ('can_manage'::text,3)
-        ), perm_edges(tail_uuid, head_uuid, val, follow, trashed) AS (
-         SELECT links.tail_uuid,
-            links.head_uuid,
-            pv.val,
-            ((pv.val = 3) OR (groups.uuid IS NOT NULL)) AS follow,
-            (0)::smallint AS trashed,
-            (0)::smallint AS followtrash
-           FROM ((public.links
-             LEFT JOIN perm_value pv ON ((pv.name = (links.name)::text)))
-             LEFT JOIN public.groups ON (((pv.val < 3) AND ((groups.uuid)::text = (links.head_uuid)::text))))
-          WHERE ((links.link_class)::text = 'permission'::text)
-        UNION ALL
-         SELECT groups.owner_uuid,
-            groups.uuid,
-            3,
-            true AS bool,
-                CASE
-                    WHEN ((groups.trash_at IS NOT NULL) AND (groups.trash_at < clock_timestamp())) THEN 1
-                    ELSE 0
-                END AS "case",
-            1
-           FROM public.groups
-        ), perm(val, follow, user_uuid, target_uuid, trashed) AS (
-         SELECT (3)::smallint AS val,
-            true AS follow,
-            (users.uuid)::character varying(32) AS user_uuid,
-            (users.uuid)::character varying(32) AS target_uuid,
-            (0)::smallint AS trashed
-           FROM public.users
-        UNION
-         SELECT (LEAST((perm_1.val)::integer, edges.val))::smallint AS val,
-            edges.follow,
-            perm_1.user_uuid,
-            (edges.head_uuid)::character varying(32) AS target_uuid,
-            ((GREATEST((perm_1.trashed)::integer, edges.trashed) * edges.followtrash))::smallint AS trashed
-           FROM (perm perm_1
-             JOIN perm_edges edges ON ((perm_1.follow AND ((edges.tail_uuid)::text = (perm_1.target_uuid)::text))))
-        )
- SELECT perm.user_uuid,
-    perm.target_uuid,
-    max(perm.val) AS perm_level,
-        CASE perm.follow
-            WHEN true THEN perm.target_uuid
-            ELSE NULL::character varying
-        END AS target_owner_uuid,
-    max(perm.trashed) AS trashed
-   FROM perm
-  GROUP BY perm.user_uuid, perm.target_uuid,
-        CASE perm.follow
-            WHEN true THEN perm.target_uuid
-            ELSE NULL::character varying
-        END
-  WITH NO DATA;
-
-
 --
 -- Name: nodes; Type: TABLE; Schema: public; Owner: -
 --
@@ -851,31 +985,66 @@ ALTER SEQUENCE public.nodes_id_seq OWNED BY public.nodes.id;
 
 
 --
--- Name: permission_refresh_lock; Type: TABLE; Schema: public; Owner: -
+-- Name: users; Type: TABLE; Schema: public; Owner: -
 --
 
-CREATE TABLE public.permission_refresh_lock (
-    id integer NOT NULL
+CREATE TABLE public.users (
+    id integer NOT NULL,
+    uuid character varying(255),
+    owner_uuid character varying(255) NOT NULL,
+    created_at timestamp without time zone NOT NULL,
+    modified_by_client_uuid character varying(255),
+    modified_by_user_uuid character varying(255),
+    modified_at timestamp without time zone,
+    email character varying(255),
+    first_name character varying(255),
+    last_name character varying(255),
+    identity_url character varying(255),
+    is_admin boolean,
+    prefs text,
+    updated_at timestamp without time zone NOT NULL,
+    default_owner_uuid character varying(255),
+    is_active boolean DEFAULT false,
+    username character varying(255),
+    redirect_to_user_uuid character varying
 );
 
 
 --
--- Name: permission_refresh_lock_id_seq; Type: SEQUENCE; Schema: public; Owner: -
---
-
-CREATE SEQUENCE public.permission_refresh_lock_id_seq
-    START WITH 1
-    INCREMENT BY 1
-    NO MINVALUE
-    NO MAXVALUE
-    CACHE 1;
-
-
---
--- Name: permission_refresh_lock_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: -
---
-
-ALTER SEQUENCE public.permission_refresh_lock_id_seq OWNED BY public.permission_refresh_lock.id;
+-- Name: permission_graph_edges; Type: VIEW; Schema: public; Owner: -
+--
+
+CREATE VIEW public.permission_graph_edges AS
+ SELECT groups.owner_uuid AS tail_uuid,
+    groups.uuid AS head_uuid,
+    3 AS val,
+    groups.uuid AS edge_id
+   FROM public.groups
+UNION ALL
+ SELECT users.owner_uuid AS tail_uuid,
+    users.uuid AS head_uuid,
+    3 AS val,
+    users.uuid AS edge_id
+   FROM public.users
+UNION ALL
+ SELECT users.uuid AS tail_uuid,
+    users.uuid AS head_uuid,
+    3 AS val,
+    ''::character varying AS edge_id
+   FROM public.users
+UNION ALL
+ SELECT links.tail_uuid,
+    links.head_uuid,
+        CASE
+            WHEN ((links.name)::text = 'can_read'::text) THEN 1
+            WHEN ((links.name)::text = 'can_login'::text) THEN 1
+            WHEN ((links.name)::text = 'can_write'::text) THEN 2
+            WHEN ((links.name)::text = 'can_manage'::text) THEN 3
+            ELSE 0
+        END AS val,
+    links.uuid AS edge_id
+   FROM public.links
+  WHERE ((links.link_class)::text = 'permission'::text);
 
 
 --
@@ -1079,6 +1248,16 @@ CREATE SEQUENCE public.traits_id_seq
 ALTER SEQUENCE public.traits_id_seq OWNED BY public.traits.id;
 
 
+--
+-- Name: trashed_groups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.trashed_groups (
+    group_uuid character varying,
+    trash_at timestamp without time zone
+);
+
+
 --
 -- Name: users_id_seq; Type: SEQUENCE; Schema: public; Owner: -
 --
@@ -1277,13 +1456,6 @@ ALTER TABLE ONLY public.logs ALTER COLUMN id SET DEFAULT nextval('public.logs_id
 ALTER TABLE ONLY public.nodes ALTER COLUMN id SET DEFAULT nextval('public.nodes_id_seq'::regclass);
 
 
---
--- Name: permission_refresh_lock id; Type: DEFAULT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.permission_refresh_lock ALTER COLUMN id SET DEFAULT nextval('public.permission_refresh_lock_id_seq'::regclass);
-
-
 --
 -- Name: pipeline_instances id; Type: DEFAULT; Schema: public; Owner: -
 --
@@ -1468,14 +1640,6 @@ ALTER TABLE ONLY public.nodes
     ADD CONSTRAINT nodes_pkey PRIMARY KEY (id);
 
 
---
--- Name: permission_refresh_lock permission_refresh_lock_pkey; Type: CONSTRAINT; Schema: public; Owner: -
---
-
-ALTER TABLE ONLY public.permission_refresh_lock
-    ADD CONSTRAINT permission_refresh_lock_pkey PRIMARY KEY (id);
-
-
 --
 -- Name: pipeline_instances pipeline_instances_pkey; Type: CONSTRAINT; Schema: public; Owner: -
 --
@@ -2513,6 +2677,13 @@ CREATE INDEX index_traits_on_owner_uuid ON public.traits USING btree (owner_uuid
 CREATE UNIQUE INDEX index_traits_on_uuid ON public.traits USING btree (uuid);
 
 
+--
+-- Name: index_trashed_groups_on_group_uuid; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE UNIQUE INDEX index_trashed_groups_on_group_uuid ON public.trashed_groups USING btree (group_uuid);
+
+
 --
 -- Name: index_users_on_created_at; Type: INDEX; Schema: public; Owner: -
 --
@@ -2703,17 +2874,17 @@ CREATE INDEX nodes_search_index ON public.nodes USING btree (uuid, owner_uuid, m
 
 
 --
--- Name: permission_target_trashed; Type: INDEX; Schema: public; Owner: -
+-- Name: permission_target; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE INDEX permission_target_trashed ON public.materialized_permission_view USING btree (trashed, target_uuid);
+CREATE INDEX permission_target ON public.materialized_permissions USING btree (target_uuid);
 
 
 --
--- Name: permission_target_user_trashed_level; Type: INDEX; Schema: public; Owner: -
+-- Name: permission_user_target; Type: INDEX; Schema: public; Owner: -
 --
 
-CREATE INDEX permission_target_user_trashed_level ON public.materialized_permission_view USING btree (user_uuid, trashed, perm_level);
+CREATE UNIQUE INDEX permission_user_target ON public.materialized_permissions USING btree (user_uuid, target_uuid);
 
 
 --
@@ -3024,6 +3195,8 @@ INSERT INTO "schema_migrations" (version) VALUES
 ('20190523180148'),
 ('20190808145904'),
 ('20190809135453'),
-('20190905151603');
+('20190905151603'),
+('20200501150153'),
+('20200602141328');
 
 
diff --git a/services/api/lib/20200501150153_permission_table_constants.rb b/services/api/lib/20200501150153_permission_table_constants.rb
new file mode 100644
index 0000000000..6e43a628c7
--- /dev/null
+++ b/services/api/lib/20200501150153_permission_table_constants.rb
@@ -0,0 +1,85 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+# These constants are used in both
+# db/migrate/20200501150153_permission_table and update_permissions
+#
+# This file allows them to be easily imported by both to avoid duplication.
+#
+# Don't mess with this!  Any changes will affect both the current
+# update_permissions and the past migration.  If you are tinkering
+# with the permission system and need to change how
+# PERM_QUERY_TEMPLATE, refresh_trashed or refresh_permissions works,
+# you should make a new file with your modified functions and have
+# update_permissions reference that file instead.
+
+PERMISSION_VIEW = "materialized_permissions"
+
+TRASHED_GROUPS = "trashed_groups"
+
+# We need to use this parameterized query in a few different places,
+# including as a subquery in a larger query.
+#
+# There's basically two options, the way I did this originally was to
+# put this in a postgres function and do a lateral join over it.
+# However, postgres functions impose an optimization barrier, and
+# possibly have other overhead with temporary tables, so I ended up
+# going with the brute force approach of inlining the whole thing.
+#
+# The two substitutions are "base_case" which determines the initial
+# set of permission origins and "edge_perm" which is used to ensure
+# that the new permission takes precedence over the one in the edges
+# table (but some queries don't need that.)
+#
+PERM_QUERY_TEMPLATE = %{
+WITH RECURSIVE
+        traverse_graph(origin_uuid, target_uuid, val, traverse_owned, starting_set) as (
+            %{base_case}
+          union
+            (select traverse_graph.origin_uuid,
+                    edges.head_uuid,
+                      least(%{edge_perm},
+                            traverse_graph.val),
+                    should_traverse_owned(edges.head_uuid, edges.val),
+                    false
+             from permission_graph_edges as edges, traverse_graph
+             where traverse_graph.target_uuid = edges.tail_uuid
+             and (edges.tail_uuid like '_____-j7d0g-_______________' or
+                  traverse_graph.starting_set)))
+        select traverse_graph.origin_uuid, target_uuid, max(val) as val, bool_or(traverse_owned) as traverse_owned from traverse_graph
+        group by (traverse_graph.origin_uuid, target_uuid)
+}
+
+def refresh_trashed
+  ActiveRecord::Base.transaction do
+    ActiveRecord::Base.connection.execute("LOCK TABLE #{TRASHED_GROUPS}")
+    ActiveRecord::Base.connection.execute("DELETE FROM #{TRASHED_GROUPS}")
+
+    # Helper populate trashed_groups table. This starts with
+    #   each group owned by a user and computes the subtree under that
+    #   group to find any groups that are trashed.
+    ActiveRecord::Base.connection.execute(%{
+INSERT INTO #{TRASHED_GROUPS}
+select ps.target_uuid as group_uuid, ps.trash_at from groups,
+  lateral project_subtree_with_trash_at(groups.uuid, groups.trash_at) ps
+  where groups.owner_uuid like '_____-tpzed-_______________'
+})
+  end
+end
+
+def refresh_permissions
+  ActiveRecord::Base.transaction do
+    ActiveRecord::Base.connection.execute("LOCK TABLE #{PERMISSION_VIEW}")
+    ActiveRecord::Base.connection.execute("DELETE FROM #{PERMISSION_VIEW}")
+
+    ActiveRecord::Base.connection.execute %{
+INSERT INTO materialized_permissions
+    #{PERM_QUERY_TEMPLATE % {:base_case => %{
+        select uuid, uuid, 3, true, true from users
+},
+:edge_perm => 'edges.val'
+} }
+}, "refresh_permission_view.do"
+  end
+end
diff --git a/services/api/lib/current_api_client.rb b/services/api/lib/current_api_client.rb
index c7b48c0cdd..98112c858b 100644
--- a/services/api/lib/current_api_client.rb
+++ b/services/api/lib/current_api_client.rb
@@ -90,7 +90,8 @@ module CurrentApiClient
         ActiveRecord::Base.transaction do
           Group.where(uuid: system_group_uuid).
             first_or_create!(name: "System group",
-                             description: "System group") do |g|
+                             description: "System group",
+                             group_class: "role") do |g|
             g.save!
             User.all.collect(&:uuid).each do |user_uuid|
               Link.create!(link_class: 'permission',
@@ -188,7 +189,7 @@ module CurrentApiClient
     end
   end
 
-  def empty_collection_uuid
+  def empty_collection_pdh
     'd41d8cd98f00b204e9800998ecf8427e+0'
   end
 
@@ -197,8 +198,16 @@ module CurrentApiClient
       act_as_system_user do
         ActiveRecord::Base.transaction do
           Collection.
-            where(portable_data_hash: empty_collection_uuid).
-            first_or_create!(manifest_text: '', owner_uuid: anonymous_group.uuid)
+            where(portable_data_hash: empty_collection_pdh).
+            first_or_create(manifest_text: '', owner_uuid: system_user.uuid, name: "empty collection") do |c|
+            c.save!
+            Link.where(tail_uuid: anonymous_group.uuid,
+                       head_uuid: c.uuid,
+                       link_class: 'permission',
+                       name: 'can_read').
+                  first_or_create!
+            c
+          end
         end
       end
     end
diff --git a/services/api/lib/db_current_time.rb b/services/api/lib/db_current_time.rb
index fdb6641521..5e1634ecb9 100644
--- a/services/api/lib/db_current_time.rb
+++ b/services/api/lib/db_current_time.rb
@@ -3,9 +3,13 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 module DbCurrentTime
-  CURRENT_TIME_SQL = "SELECT clock_timestamp()"
+  CURRENT_TIME_SQL = "SELECT clock_timestamp() AT TIME ZONE 'UTC'"
 
   def db_current_time
-    Time.parse(ActiveRecord::Base.connection.select_value(CURRENT_TIME_SQL)).to_time
+    Time.parse(ActiveRecord::Base.connection.select_value(CURRENT_TIME_SQL) + " +0000")
+  end
+
+  def db_transaction_time
+    Time.parse(ActiveRecord::Base.connection.select_value("SELECT current_timestamp AT TIME ZONE 'UTC'") + " +0000")
   end
 end
diff --git a/services/api/lib/fix_roles_projects.rb b/services/api/lib/fix_roles_projects.rb
new file mode 100644
index 0000000000..5bb013c9ad
--- /dev/null
+++ b/services/api/lib/fix_roles_projects.rb
@@ -0,0 +1,78 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'update_permissions'
+
+include CurrentApiClient
+
+def fix_roles_projects
+  batch_update_permissions do
+    # This migration is not reversible.  However, the behavior it
+    # enforces is backwards-compatible, and most of the time there
+    # shouldn't be anything to do at all.
+    act_as_system_user do
+      ActiveRecord::Base.transaction do
+        Group.where("group_class != 'project' or group_class is null").each do |g|
+          # 1) any group not group_class != project becomes a 'role' (both empty and invalid groups)
+          old_owner = g.owner_uuid
+          g.owner_uuid = system_user_uuid
+          g.group_class = 'role'
+          g.save_with_unique_name!
+
+          if old_owner != system_user_uuid
+            # 2) Ownership of a role becomes a can_manage link
+            Link.new(link_class: 'permission',
+                         name: 'can_manage',
+                         tail_uuid: old_owner,
+                         head_uuid: g.uuid).
+              save!(validate: false)
+          end
+        end
+
+        ActiveRecord::Base.descendants.reject(&:abstract_class?).each do |klass|
+          next if [ApiClientAuthorization,
+                   AuthorizedKey,
+                   Log,
+                   Group].include?(klass)
+          next if !klass.columns.collect(&:name).include?('owner_uuid')
+
+          # 3) If a role owns anything, give it to system user and it
+          # becomes a can_manage link
+          klass.joins("join groups on groups.uuid=#{klass.table_name}.owner_uuid and groups.group_class='role'").each do |owned|
+            Link.new(link_class: 'permission',
+                     name: 'can_manage',
+                     tail_uuid: owned.owner_uuid,
+                     head_uuid: owned.uuid).
+              save!(validate: false)
+            owned.owner_uuid = system_user_uuid
+            owned.save_with_unique_name!
+          end
+        end
+
+        Group.joins("join groups as g2 on g2.uuid=groups.owner_uuid and g2.group_class='role'").each do |owned|
+          Link.new(link_class: 'permission',
+                       name: 'can_manage',
+                       tail_uuid: owned.owner_uuid,
+                       head_uuid: owned.uuid).
+            save!(validate: false)
+          owned.owner_uuid = system_user_uuid
+          owned.save_with_unique_name!
+        end
+
+        # 4) Projects can't have outgoing permission links.  Just
+        # print a warning and delete them.
+        q = ActiveRecord::Base.connection.exec_query %{
+select links.uuid from links, groups where groups.uuid = links.tail_uuid and
+       links.link_class = 'permission' and groups.group_class = 'project'
+}
+        q.each do |lu|
+          ln = Link.find_by_uuid(lu['uuid'])
+          puts "WARNING: Projects cannot have outgoing permission links, removing '#{ln.name}' link #{ln.uuid} from #{ln.tail_uuid} to #{ln.head_uuid}"
+          Rails.logger.warn "Projects cannot have outgoing permission links, removing '#{ln.name}' link #{ln.uuid} from #{ln.tail_uuid} to #{ln.head_uuid}"
+          ln.destroy!
+        end
+      end
+    end
+  end
+end
diff --git a/services/api/lib/refresh_permission_view.rb b/services/api/lib/refresh_permission_view.rb
deleted file mode 100644
index 5d6081f262..0000000000
--- a/services/api/lib/refresh_permission_view.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-PERMISSION_VIEW = "materialized_permission_view"
-
-def do_refresh_permission_view
-  ActiveRecord::Base.transaction do
-    ActiveRecord::Base.connection.execute("LOCK TABLE permission_refresh_lock")
-    ActiveRecord::Base.connection.execute("REFRESH MATERIALIZED VIEW #{PERMISSION_VIEW}")
-  end
-end
-
-def refresh_permission_view(async=false)
-  if async and Rails.configuration.API.AsyncPermissionsUpdateInterval > 0
-    exp = Rails.configuration.API.AsyncPermissionsUpdateInterval.seconds
-    need = false
-    Rails.cache.fetch('AsyncRefreshPermissionView', expires_in: exp) do
-      need = true
-    end
-    if need
-      # Schedule a new permission update and return immediately
-      Thread.new do
-        Thread.current.abort_on_exception = false
-        begin
-          sleep(exp)
-          Rails.cache.delete('AsyncRefreshPermissionView')
-          do_refresh_permission_view
-        rescue => e
-          Rails.logger.error "Updating permission view: #{e}\n#{e.backtrace.join("\n\t")}"
-        ensure
-          ActiveRecord::Base.connection.close
-        end
-      end
-      true
-    end
-  else
-    do_refresh_permission_view
-  end
-end
diff --git a/services/api/lib/update_permissions.rb b/services/api/lib/update_permissions.rb
new file mode 100644
index 0000000000..7b1b900cac
--- /dev/null
+++ b/services/api/lib/update_permissions.rb
@@ -0,0 +1,218 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require '20200501150153_permission_table_constants'
+
+REVOKE_PERM = 0
+CAN_MANAGE_PERM = 3
+
+def update_permissions perm_origin_uuid, starting_uuid, perm_level, edge_id=nil
+  return if Thread.current[:suppress_update_permissions]
+
+  #
+  # Update a subset of the permission table affected by adding or
+  # removing a particular permission relationship (ownership or a
+  # permission link).
+  #
+  # perm_origin_uuid: This is the object that 'gets' the permission.
+  # It is the owner_uuid or tail_uuid.
+  #
+  # starting_uuid: The object we are computing permission for (or head_uuid)
+  #
+  # perm_level: The level of permission that perm_origin_uuid gets for starting_uuid.
+  #
+  # perm_level is a number from 0-3
+  #   can_read=1
+  #   can_write=2
+  #   can_manage=3
+  #   or call with perm_level=0 to revoke permissions
+  #
+  # check: for testing/debugging, compare the result of the
+  # incremental update against a full table recompute.  Throws an
+  # error if the contents are not identical (ie they produce different
+  # permission results)
+
+  # Theory of operation
+  #
+  # Give a change in a specific permission relationship, we recompute
+  # the set of permissions (for all users) that could possibly be
+  # affected by that relationship.  For example, if a project is
+  # shared with another user, we recompute all permissions for all
+  # projects in the hierarchy.  This returns a set of updated
+  # permissions, which we stash in a temporary table.
+  #
+  # Then, for each user_uuid/target_uuid in the updated permissions
+  # result set we insert/update a permission row in
+  # materialized_permissions, and delete any rows that exist in
+  # materialized_permissions that are not in the result set or have
+  # perm_level=0.
+  #
+  # see db/migrate/20200501150153_permission_table.rb for details on
+  # how the permissions are computed.
+
+  if edge_id.nil?
+    # For changes of ownership, edge_id is starting_uuid.  In turns
+    # out most invocations of update_permissions are for changes of
+    # ownership, so make this parameter optional to reduce
+    # clutter.
+    # For permission links, the uuid of the link object will be passed in for edge_id.
+    edge_id = starting_uuid
+  end
+
+  ActiveRecord::Base.transaction do
+
+    # "Conflicts with the ROW EXCLUSIVE, SHARE UPDATE EXCLUSIVE, SHARE
+    # ROW EXCLUSIVE, EXCLUSIVE, and ACCESS EXCLUSIVE lock modes. This
+    # mode protects a table against concurrent data changes."
+    ActiveRecord::Base.connection.execute "LOCK TABLE #{PERMISSION_VIEW} in SHARE MODE"
+
+    # Workaround for
+    # BUG #15160: planner overestimates number of rows in join when there are more than 200 rows coming from CTE
+    # https://www.postgresql.org/message-id/152395805004.19366.3107109716821067806@wrigleys.postgresql.org
+    #
+    # For a crucial join in the compute_permission_subgraph() query, the
+    # planner mis-estimates the number of rows in a Common Table
+    # Expression (CTE, this is a subquery in a WITH clause) and as a
+    # result it chooses the wrong join order.  The join starts with the
+    # permissions table because it mistakenly thinks
+    # count(materalized_permissions) < count(new computed permissions)
+    # when actually it is the other way around.
+    #
+    # Because of the incorrect join order, it choose the wrong join
+    # strategy (merge join, which works best when two tables are roughly
+    # the same size).  As a workaround, we can tell it not to use that
+    # join strategy, this causes it to pick hash join instead, which
+    # turns out to be a bit better.  However, because the join order is
+    # still wrong, we don't get the full benefit of the index.
+    #
+    # This is very unfortunate because it makes the query performance
+    # dependent on the size of the materalized_permissions table, when
+    # the goal of this design was to make permission updates scale-free
+    # and only depend on the number of permissions affected and not the
+    # total table size.  In several hours of researching I wasn't able
+    # to find a way to force the correct join order, so I'm calling it
+    # here and I have to move on.
+    #
+    # This is apparently addressed in Postgres 12, but I developed &
+    # tested this on Postgres 9.6, so in the future we should reevaluate
+    # the performance & query plan on Postgres 12.
+    #
+    # https://git.furworks.de/opensourcemirror/postgresql/commit/a314c34079cf06d05265623dd7c056f8fa9d577f
+    #
+    # Disable merge join for just this query (also local for this transaction), then reenable it.
+    ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to false;"
+
+    temptable_perms = "temp_perms_#{rand(2**64).to_s(10)}"
+    ActiveRecord::Base.connection.exec_query %{
+create temporary table #{temptable_perms} on commit drop
+as select * from compute_permission_subgraph($1, $2, $3, $4)
+},
+                                             'update_permissions.select',
+                                             [[nil, perm_origin_uuid],
+                                              [nil, starting_uuid],
+                                              [nil, perm_level],
+                                              [nil, edge_id]]
+
+    ActiveRecord::Base.connection.exec_query "SET LOCAL enable_mergejoin to true;"
+
+    ActiveRecord::Base.connection.exec_delete %{
+delete from #{PERMISSION_VIEW} where
+  target_uuid in (select target_uuid from #{temptable_perms}) and
+  not exists (select 1 from #{temptable_perms}
+              where target_uuid=#{PERMISSION_VIEW}.target_uuid and
+                    user_uuid=#{PERMISSION_VIEW}.user_uuid and
+                    val>0)
+},
+                                              "update_permissions.delete"
+
+    ActiveRecord::Base.connection.exec_query %{
+insert into #{PERMISSION_VIEW} (user_uuid, target_uuid, perm_level, traverse_owned)
+  select user_uuid, target_uuid, val as perm_level, traverse_owned from #{temptable_perms} where val>0
+on conflict (user_uuid, target_uuid) do update set perm_level=EXCLUDED.perm_level, traverse_owned=EXCLUDED.traverse_owned;
+},
+                                             "update_permissions.insert"
+
+    if perm_level>0
+      check_permissions_against_full_refresh
+    end
+  end
+end
+
+
+def check_permissions_against_full_refresh
+  # No-op except when running tests
+  return unless Rails.env == 'test' and !Thread.current[:no_check_permissions_against_full_refresh] and !Thread.current[:suppress_update_permissions]
+
+  # For checking correctness of the incremental permission updates.
+  # Check contents of the current 'materialized_permission' table
+  # against a from-scratch permission refresh.
+
+  q1 = ActiveRecord::Base.connection.exec_query %{
+select user_uuid, target_uuid, perm_level, traverse_owned from #{PERMISSION_VIEW}
+order by user_uuid, target_uuid
+}, "check_permissions_against_full_refresh.permission_table"
+
+  q2 = ActiveRecord::Base.connection.exec_query %{
+    select pq.origin_uuid as user_uuid, target_uuid, pq.val as perm_level, pq.traverse_owned from (
+    #{PERM_QUERY_TEMPLATE % {:base_case => %{
+        select uuid, uuid, 3, true, true from users
+},
+:edge_perm => 'edges.val'
+} }) as pq order by origin_uuid, target_uuid
+}, "check_permissions_against_full_refresh.full_recompute"
+
+  if q1.count != q2.count
+    puts "Didn't match incremental+: #{q1.count} != full refresh-: #{q2.count}"
+  end
+
+  if q1.count > q2.count
+    q1.each_with_index do |r, i|
+      if r != q2[i]
+        puts "+#{r}\n-#{q2[i]}"
+        raise "Didn't match"
+      end
+    end
+  else
+    q2.each_with_index do |r, i|
+      if r != q1[i]
+        puts "+#{q1[i]}\n-#{r}"
+        raise "Didn't match"
+      end
+    end
+  end
+end
+
+def skip_check_permissions_against_full_refresh
+  check_perm_was = Thread.current[:no_check_permissions_against_full_refresh]
+  Thread.current[:no_check_permissions_against_full_refresh] = true
+  begin
+    yield
+  ensure
+    Thread.current[:no_check_permissions_against_full_refresh] = check_perm_was
+  end
+end
+
+def batch_update_permissions
+  check_perm_was = Thread.current[:suppress_update_permissions]
+  Thread.current[:suppress_update_permissions] = true
+  begin
+    yield
+  ensure
+    Thread.current[:suppress_update_permissions] = check_perm_was
+    refresh_permissions
+  end
+end
+
+# Used to account for permissions that a user gains by having
+# can_manage on another user.
+#
+# note: in theory a user could have can_manage access to a user
+# through multiple levels, that isn't handled here (would require a
+# recursive query).  I think that's okay because users getting
+# transitive access through "can_manage" on a user is is rarely/never
+# used feature and something we probably want to deprecate and remove.
+USER_UUIDS_SUBQUERY_TEMPLATE = %{
+select target_uuid from materialized_permissions where user_uuid in (%{user})
+and target_uuid like '_____-tpzed-_______________' and traverse_owned=true and perm_level >= %{perm_level}
+}
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 1581039bb3..a16ee8763f 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -199,7 +199,6 @@ unlinked_docker_image:
 empty:
   uuid: zzzzz-4zz18-gs9ooj1h9sd5mde
   current_version_uuid: zzzzz-4zz18-gs9ooj1h9sd5mde
-  # Empty collection owned by anonymous_group is added with rake db:seed.
   portable_data_hash: d41d8cd98f00b204e9800998ecf8427e+0
   owner_uuid: zzzzz-tpzed-000000000000000
   created_at: 2014-06-11T17:22:54Z
@@ -208,7 +207,7 @@ empty:
   modified_at: 2014-06-11T17:22:54Z
   updated_at: 2014-06-11T17:22:54Z
   manifest_text: ""
-  name: empty_collection
+  name: "empty collection for python test"
 
 foo_collection_in_aproject:
   uuid: zzzzz-4zz18-fy296fx3hot09f7
diff --git a/services/api/test/fixtures/groups.yml b/services/api/test/fixtures/groups.yml
index 92a1ced528..ee0d786bbe 100644
--- a/services/api/test/fixtures/groups.yml
+++ b/services/api/test/fixtures/groups.yml
@@ -6,19 +6,33 @@ public:
   uuid: zzzzz-j7d0g-it30l961gq3t0oi
   owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
   name: Public
-  description: Public Group
+  description: Public Project
+  group_class: project
+
+public_role:
+  uuid: zzzzz-j7d0g-jt30l961gq3t0oi
+  owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
+  name: Public Role
+  description: Public Role
   group_class: role
 
 private:
   uuid: zzzzz-j7d0g-rew6elm53kancon
   owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
   name: Private
-  description: Private Group
+  description: Private Project
+  group_class: project
+
+private_role:
+  uuid: zzzzz-j7d0g-pew6elm53kancon
+  owner_uuid: zzzzz-tpzed-000000000000000
+  name: Private Role
+  description: Private Role
   group_class: role
 
 private_and_can_read_foofile:
   uuid: zzzzz-j7d0g-22xp1wpjul508rk
-  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  owner_uuid: zzzzz-tpzed-000000000000000
   name: Private and Can Read Foofile
   description: Another Private Group
   group_class: role
@@ -60,6 +74,7 @@ testusergroup_admins:
   uuid: zzzzz-j7d0g-48foin4vonvc2at
   owner_uuid: zzzzz-tpzed-000000000000000
   name: Administrators of a subset of users
+  group_class: role
 
 aproject:
   uuid: zzzzz-j7d0g-v955i6s2oi1cbso
@@ -87,7 +102,7 @@ asubproject:
 
 future_project_viewing_group:
   uuid: zzzzz-j7d0g-futrprojviewgrp
-  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  owner_uuid: zzzzz-tpzed-000000000000000
   created_at: 2014-04-21 15:37:48 -0400
   modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
   modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
@@ -106,6 +121,7 @@ bad_group_has_ownership_cycle_a:
   modified_at: 2014-05-03 18:50:08 -0400
   updated_at: 2014-05-03 18:50:08 -0400
   name: Owned by bad group b
+  group_class: project
 
 bad_group_has_ownership_cycle_b:
   uuid: zzzzz-j7d0g-0077nzts8c178lw
@@ -116,6 +132,7 @@ bad_group_has_ownership_cycle_b:
   modified_at: 2014-05-03 18:50:08 -0400
   updated_at: 2014-05-03 18:50:08 -0400
   name: Owned by bad group a
+  group_class: project
 
 anonymous_group:
   uuid: zzzzz-j7d0g-anonymouspublic
@@ -143,6 +160,7 @@ active_user_has_can_manage:
   uuid: zzzzz-j7d0g-ptt1ou6a9lxrv07
   owner_uuid: zzzzz-tpzed-d9tiejq69daie8f
   name: Active user has can_manage
+  group_class: project
 
 # Group for testing granting permission between users who share a group.
 group_for_sharing_tests:
@@ -244,17 +262,6 @@ fuse_owned_project:
   description: Test project belonging to FUSE test user
   group_class: project
 
-group_with_no_class:
-  uuid: zzzzz-j7d0g-groupwithnoclas
-  owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
-  created_at: 2014-04-21 15:37:48 -0400
-  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
-  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
-  modified_at: 2014-04-21 15:37:48 -0400
-  updated_at: 2014-04-21 15:37:48 -0400
-  name: group_with_no_class
-  description: This group has no class at all. So rude!
-
 # This wouldn't pass model validation, but it enables a workbench
 # infinite-loop test. See #4389
 project_owns_itself:
@@ -343,4 +350,4 @@ trashed_on_next_sweep:
   trash_at: 2001-01-01T00:00:00Z
   delete_at: 2038-03-01T00:00:00Z
   is_trashed: false
-  modified_at: 2001-01-01T00:00:00Z
\ No newline at end of file
+  modified_at: 2001-01-01T00:00:00Z
diff --git a/services/api/test/fixtures/links.yml b/services/api/test/fixtures/links.yml
index 2f66433379..ee5dcd3421 100644
--- a/services/api/test/fixtures/links.yml
+++ b/services/api/test/fixtures/links.yml
@@ -198,7 +198,7 @@ foo_file_readable_by_active_redundant_permission_via_private_group:
   head_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
   properties: {}
 
-foo_file_readable_by_aproject:
+foo_file_readable_by_project_viewer:
   uuid: zzzzz-o0j2j-fp1d8395ldqw22p
   owner_uuid: zzzzz-tpzed-000000000000000
   created_at: 2014-01-24 20:42:26 -0800
@@ -206,7 +206,7 @@ foo_file_readable_by_aproject:
   modified_by_user_uuid: zzzzz-tpzed-000000000000000
   modified_at: 2014-01-24 20:42:26 -0800
   updated_at: 2014-01-24 20:42:26 -0800
-  tail_uuid: zzzzz-j7d0g-v955i6s2oi1cbso
+  tail_uuid: zzzzz-tpzed-projectviewer1a
   link_class: permission
   name: can_read
   head_uuid: zzzzz-4zz18-znfnqtbbv4spc3w
@@ -1111,3 +1111,17 @@ tagged_collection_readable_by_spectator:
   name: can_read
   head_uuid: zzzzz-4zz18-taggedcolletion
   properties: {}
+
+active_manages_viewing_group:
+  uuid: zzzzz-o0j2j-activemanagesvi
+  owner_uuid: zzzzz-tpzed-000000000000000
+  created_at: 2014-01-24 20:42:26 -0800
+  modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+  modified_by_user_uuid: zzzzz-tpzed-000000000000000
+  modified_at: 2014-01-24 20:42:26 -0800
+  updated_at: 2014-01-24 20:42:26 -0800
+  tail_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  link_class: permission
+  name: can_manage
+  head_uuid: zzzzz-j7d0g-futrprojviewgrp
+  properties: {}
diff --git a/services/api/test/fixtures/users.yml b/services/api/test/fixtures/users.yml
index 57633a3120..14630d9efa 100644
--- a/services/api/test/fixtures/users.yml
+++ b/services/api/test/fixtures/users.yml
@@ -418,3 +418,17 @@ double_redirects_to_active:
       organization: example.com
       role: Computational biologist
     getting_started_shown: 2015-03-26 12:34:56.789000000 Z
+
+has_can_login_permission:
+  owner_uuid: zzzzz-tpzed-000000000000000
+  uuid: zzzzz-tpzed-xabcdjxw79nv3jz
+  email: can-login-user@arvados.local
+  modified_by_client_uuid: zzzzz-ozdt8-teyxzyd8qllg11h
+  modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+  first_name: Can_login
+  last_name: User
+  identity_url: https://can-login-user.openid.local
+  is_active: true
+  is_admin: false
+  modified_at: 2015-03-26 12:34:56.789000000 Z
+  username: can-login-user
diff --git a/services/api/test/functional/application_controller_test.rb b/services/api/test/functional/application_controller_test.rb
index 175a8f71ea..2cfa054448 100644
--- a/services/api/test/functional/application_controller_test.rb
+++ b/services/api/test/functional/application_controller_test.rb
@@ -100,7 +100,7 @@ class ApplicationControllerTest < ActionController::TestCase
         @controller = Arvados::V1::GroupsController.new
         authorize_with :active
         post :create, params: {
-          group: {},
+          group: {group_class: "project"},
           ensure_unique_name: boolparam
         }
         assert_response :success
@@ -113,7 +113,8 @@ class ApplicationControllerTest < ActionController::TestCase
         post :create, params: {
           group: {
             name: groups(:aproject).name,
-            owner_uuid: groups(:aproject).owner_uuid
+            owner_uuid: groups(:aproject).owner_uuid,
+            group_class: "project"
           },
           ensure_unique_name: boolparam
         }
diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index b30afd7453..26270b1c3c 100644
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -6,16 +6,16 @@ require 'test_helper'
 
 class Arvados::V1::FiltersTest < ActionController::TestCase
   test '"not in" filter passes null values' do
-    @controller = Arvados::V1::GroupsController.new
+    @controller = Arvados::V1::ContainerRequestsController.new
     authorize_with :admin
     get :index, params: {
-      filters: [ ['group_class', 'not in', ['project']] ],
-      controller: 'groups',
+      filters: [ ['container_uuid', 'not in', ['zzzzz-dz642-queuedcontainer', 'zzzzz-dz642-runningcontainr']] ],
+      controller: 'container_requests',
     }
     assert_response :success
     found = assigns(:objects)
-    assert_includes(found.collect(&:group_class), nil,
-                    "'group_class not in ['project']' filter should pass null")
+    assert_includes(found.collect(&:container_uuid), nil,
+                    "'container_uuid not in [zzzzz-dz642-queuedcontainer, zzzzz-dz642-runningcontainr]' filter should pass null")
   end
 
   test 'error message for non-array element in filters array' do
diff --git a/services/api/test/functional/arvados/v1/groups_controller_test.rb b/services/api/test/functional/arvados/v1/groups_controller_test.rb
index 30ab89c7e2..ff89cd2129 100644
--- a/services/api/test/functional/arvados/v1/groups_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/groups_controller_test.rb
@@ -29,8 +29,9 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
     end
     assert_includes group_uuids, groups(:aproject).uuid
     assert_includes group_uuids, groups(:asubproject).uuid
+    assert_includes group_uuids, groups(:private).uuid
     assert_not_includes group_uuids, groups(:system_group).uuid
-    assert_not_includes group_uuids, groups(:private).uuid
+    assert_not_includes group_uuids, groups(:private_and_can_read_foofile).uuid
   end
 
   test "get list of groups that are not projects" do
@@ -44,8 +45,6 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
     end
     assert_not_includes group_uuids, groups(:aproject).uuid
     assert_not_includes group_uuids, groups(:asubproject).uuid
-    assert_includes group_uuids, groups(:private).uuid
-    assert_includes group_uuids, groups(:group_with_no_class).uuid
   end
 
   test "get list of groups with bogus group_class" do
@@ -505,9 +504,19 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
 
   ### trashed project tests ###
 
-  [:active, :admin].each do |auth|
+  #
+  # The structure is
+  #
+  # trashed_project         (zzzzz-j7d0g-trashedproject1)
+  #   trashed_subproject    (zzzzz-j7d0g-trashedproject2)
+  #   trashed_subproject3   (zzzzz-j7d0g-trashedproject3)
+  #   zzzzz-xvhdp-cr5trashedcontr
+
+  [:active,
+   :admin].each do |auth|
     # project: to query,    to untrash,    is visible, parent contents listing success
-    [[:trashed_project,     [],                 false, true],
+    [
+     [:trashed_project,     [],                 false, true],
      [:trashed_project,     [:trashed_project], true,  true],
      [:trashed_subproject,  [],                 false, false],
      [:trashed_subproject,  [:trashed_project], true,  true],
@@ -736,20 +745,23 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
     assert_equal 0, json_response['included'].length
   end
 
-  test 'get shared, owned by non-project' do
+  test 'get shared, add permission link' do
     authorize_with :user_bar_in_sharing_group
 
     act_as_system_user do
-      Group.find_by_uuid(groups(:project_owned_by_foo).uuid).update!(owner_uuid: groups(:group_for_sharing_tests).uuid)
+      Link.create!(tail_uuid: groups(:group_for_sharing_tests).uuid,
+                   head_uuid: groups(:project_owned_by_foo).uuid,
+                   link_class: 'permission',
+                   name: 'can_manage')
     end
 
     get :shared, params: {:filters => [["group_class", "=", "project"]], :include => "owner_uuid"}
 
     assert_equal 1, json_response['items'].length
-    assert_equal json_response['items'][0]["uuid"], groups(:project_owned_by_foo).uuid
+    assert_equal groups(:project_owned_by_foo).uuid, json_response['items'][0]["uuid"]
 
     assert_equal 1, json_response['included'].length
-    assert_equal json_response['included'][0]["uuid"], groups(:group_for_sharing_tests).uuid
+    assert_equal users(:user_foo_in_sharing_group).uuid, json_response['included'][0]["uuid"]
   end
 
   ### contents with exclude_home_project
@@ -800,20 +812,23 @@ class Arvados::V1::GroupsControllerTest < ActionController::TestCase
     assert_equal 0, json_response['included'].length
   end
 
-  test 'contents, exclude home, owned by non-project' do
+  test 'contents, exclude home, add permission link' do
     authorize_with :user_bar_in_sharing_group
 
     act_as_system_user do
-      Group.find_by_uuid(groups(:project_owned_by_foo).uuid).update!(owner_uuid: groups(:group_for_sharing_tests).uuid)
+      Link.create!(tail_uuid: groups(:group_for_sharing_tests).uuid,
+                   head_uuid: groups(:project_owned_by_foo).uuid,
+                   link_class: 'permission',
+                   name: 'can_manage')
     end
 
     get :contents, params: {:include => "owner_uuid", :exclude_home_project => true}
 
     assert_equal 1, json_response['items'].length
-    assert_equal json_response['items'][0]["uuid"], groups(:project_owned_by_foo).uuid
+    assert_equal groups(:project_owned_by_foo).uuid, json_response['items'][0]["uuid"]
 
     assert_equal 1, json_response['included'].length
-    assert_equal json_response['included'][0]["uuid"], groups(:group_for_sharing_tests).uuid
+    assert_equal users(:user_foo_in_sharing_group).uuid, json_response['included'][0]["uuid"]
   end
 
   test 'contents, exclude home, with parent specified' do
diff --git a/services/api/test/functional/arvados/v1/keep_services_controller_test.rb b/services/api/test/functional/arvados/v1/keep_services_controller_test.rb
index 867ab35e79..ce1d447f16 100644
--- a/services/api/test/functional/arvados/v1/keep_services_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/keep_services_controller_test.rb
@@ -54,7 +54,7 @@ class Arvados::V1::KeepServicesControllerTest < ActionController::TestCase
       headers: auth(:active)
     assert_response :success
     json_response['items'].each do |svc|
-      url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}"
+      url = "#{svc['service_ssl_flag'] ? 'https' : 'http'}://#{svc['service_host']}:#{svc['service_port']}/"
       assert_equal true, expect_rvz.has_key?(url), "#{url} does not match any configured service: expecting #{expect_rvz}"
       rvz = expect_rvz[url]
       if rvz.is_a? String
diff --git a/services/api/test/functional/arvados/v1/repositories_controller_test.rb b/services/api/test/functional/arvados/v1/repositories_controller_test.rb
index cfcd917d65..84bd846c91 100644
--- a/services/api/test/functional/arvados/v1/repositories_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/repositories_controller_test.rb
@@ -150,7 +150,7 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase
   test "get_all_permissions obeys group permissions" do
     act_as_user system_user do
       r = Repository.create!(name: 'admin/groupcanwrite', owner_uuid: users(:admin).uuid)
-      g = Group.create!(group_class: 'group', name: 'repo-writers')
+      g = Group.create!(group_class: 'role', name: 'repo-writers')
       u1 = users(:active)
       u2 = users(:spectator)
       Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_manage')
@@ -158,7 +158,7 @@ class Arvados::V1::RepositoriesControllerTest < ActionController::TestCase
       Link.create!(tail_uuid: u2.uuid, head_uuid: g.uuid, link_class: 'permission', name: 'can_read')
 
       r = Repository.create!(name: 'admin/groupreadonly', owner_uuid: users(:admin).uuid)
-      g = Group.create!(group_class: 'group', name: 'repo-readers')
+      g = Group.create!(group_class: 'role', name: 'repo-readers')
       u1 = users(:active)
       u2 = users(:spectator)
       Link.create!(tail_uuid: g.uuid, head_uuid: r.uuid, link_class: 'permission', name: 'can_read')
diff --git a/services/api/test/functional/arvados/v1/users_controller_test.rb b/services/api/test/functional/arvados/v1/users_controller_test.rb
index 817a1c9ef9..0ce9f1137f 100644
--- a/services/api/test/functional/arvados/v1/users_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/users_controller_test.rb
@@ -660,7 +660,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
 
   test "non-admin user gets only safe attributes from users#show" do
     g = act_as_system_user do
-      create :group
+      create :group, group_class: "role"
     end
     users = create_list :active_user, 2, join_groups: [g]
     token = create :token, user: users[0]
@@ -672,7 +672,7 @@ class Arvados::V1::UsersControllerTest < ActionController::TestCase
   [2, 4].each do |limit|
     test "non-admin user can limit index to #{limit}" do
       g = act_as_system_user do
-        create :group
+        create :group, group_class: "role"
       end
       users = create_list :active_user, 4, join_groups: [g]
       token = create :token, user: users[0]
diff --git a/services/api/test/integration/groups_test.rb b/services/api/test/integration/groups_test.rb
index eb97fc1f49..7021761278 100644
--- a/services/api/test/integration/groups_test.rb
+++ b/services/api/test/integration/groups_test.rb
@@ -193,23 +193,28 @@ class NonTransactionalGroupsTest < ActionDispatch::IntegrationTest
     assert_response :success
   end
 
-  test "create request with async=true defers permissions update" do
+  test "create request with async=true does not defer permissions update" do
     Rails.configuration.API.AsyncPermissionsUpdateInterval = 1 # second
     name = "Random group #{rand(1000)}"
     assert_equal nil, Group.find_by_name(name)
 
+    # Following the implementation of incremental permission updates
+    # (#16007) the async flag is now a no-op.  Permission changes are
+    # visible immediately.
+
     # Trigger the asynchronous permission update by using async=true parameter.
     post "/arvados/v1/groups",
       params: {
         group: {
-          name: name
+          name: name,
+          group_class: "project"
         },
         async: true
       },
       headers: auth(:active)
     assert_response 202
 
-    # The group exists on the database, but it's not accessible yet.
+    # The group exists in the database
     assert_not_nil Group.find_by_name(name)
     get "/arvados/v1/groups",
       params: {
@@ -218,7 +223,7 @@ class NonTransactionalGroupsTest < ActionDispatch::IntegrationTest
       },
       headers: auth(:active)
     assert_response 200
-    assert_equal 0, json_response['items_available']
+    assert_equal 1, json_response['items_available']
 
     # Wait a bit and try again.
     sleep(1)
diff --git a/services/api/test/integration/permissions_test.rb b/services/api/test/integration/permissions_test.rb
index eec41aa085..66a62543bb 100644
--- a/services/api/test/integration/permissions_test.rb
+++ b/services/api/test/integration/permissions_test.rb
@@ -6,7 +6,6 @@ require 'test_helper'
 
 class PermissionsTest < ActionDispatch::IntegrationTest
   include DbCurrentTime
-  include CurrentApiClient  # for empty_collection
   fixtures :users, :groups, :api_client_authorizations, :collections
 
   test "adding and removing direct can_read links" do
@@ -88,7 +87,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
           tail_uuid: users(:spectator).uuid,
           link_class: 'permission',
           name: 'can_read',
-          head_uuid: groups(:private).uuid,
+          head_uuid: groups(:private_role).uuid,
           properties: {}
         }
       },
@@ -106,7 +105,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
       params: {
         :format => :json,
         :link => {
-          tail_uuid: groups(:private).uuid,
+          tail_uuid: groups(:private_role).uuid,
           link_class: 'permission',
           name: 'can_read',
           head_uuid: collections(:foo_file).uuid,
@@ -150,7 +149,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
       params: {
         :format => :json,
         :link => {
-          tail_uuid: groups(:private).uuid,
+          tail_uuid: groups(:private_role).uuid,
           link_class: 'permission',
           name: 'can_read',
           head_uuid: collections(:foo_file).uuid,
@@ -174,7 +173,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
           tail_uuid: users(:spectator).uuid,
           link_class: 'permission',
           name: 'can_read',
-          head_uuid: groups(:private).uuid,
+          head_uuid: groups(:private_role).uuid,
           properties: {}
         }
       },
@@ -217,7 +216,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
           tail_uuid: users(:spectator).uuid,
           link_class: 'permission',
           name: 'can_read',
-          head_uuid: groups(:private).uuid,
+          head_uuid: groups(:private_role).uuid,
           properties: {}
         }
       },
@@ -229,7 +228,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
       params: {
         :format => :json,
         :link => {
-          tail_uuid: groups(:private).uuid,
+          tail_uuid: groups(:private_role).uuid,
           link_class: 'permission',
           name: 'can_read',
           head_uuid: groups(:empty_lonely_group).uuid,
@@ -441,7 +440,7 @@ class PermissionsTest < ActionDispatch::IntegrationTest
   test "active user can read the empty collection" do
     # The active user should be able to read the empty collection.
 
-    get("/arvados/v1/collections/#{empty_collection_uuid}",
+    get("/arvados/v1/collections/#{empty_collection_pdh}",
       params: {:format => :json},
       headers: auth(:active))
     assert_response :success
diff --git a/services/api/test/performance/permission_test.rb b/services/api/test/performance/permission_test.rb
index a0605f97e7..e5d62cf4cf 100644
--- a/services/api/test/performance/permission_test.rb
+++ b/services/api/test/performance/permission_test.rb
@@ -24,7 +24,7 @@ class PermissionPerfTest < ActionDispatch::IntegrationTest
     act_as_system_user do
       puts("Time spent creating records:", Benchmark.measure do
              ActiveRecord::Base.transaction do
-               root = Group.create!(owner_uuid: users(:permission_perftest).uuid)
+               root = Group.create!(owner_uuid: users(:permission_perftest).uuid, group_class: "project")
                n += 1
                a = create_eight root.uuid
                n += 8
@@ -40,7 +40,7 @@ class PermissionPerfTest < ActionDispatch::IntegrationTest
                    end
                  end
                end
-               User.invalidate_permissions_cache
+               refresh_permissions
              end
            end)
     end
diff --git a/services/api/test/test_helper.rb b/services/api/test/test_helper.rb
index 5747a85cf5..c99a57aaff 100644
--- a/services/api/test/test_helper.rb
+++ b/services/api/test/test_helper.rb
@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: AGPL-3.0
 
+require 'update_permissions'
+
 ENV["RAILS_ENV"] = "test"
 unless ENV["NO_COVERAGE_TEST"]
   begin
@@ -207,4 +209,5 @@ class ActionDispatch::IntegrationTest
 end
 
 # Ensure permissions are computed from the test fixtures.
-User.invalidate_permissions_cache
+refresh_permissions
+refresh_trashed
diff --git a/services/api/test/unit/arvados_model_test.rb b/services/api/test/unit/arvados_model_test.rb
index d447c76c6d..c1db8c8b5d 100644
--- a/services/api/test/unit/arvados_model_test.rb
+++ b/services/api/test/unit/arvados_model_test.rb
@@ -97,7 +97,7 @@ class ArvadosModelTest < ActiveSupport::TestCase
     while longstring.length < 2**16
       longstring = longstring + longstring
     end
-    g = Group.create! name: 'Has a long description', description: longstring
+    g = Group.create! name: 'Has a long description', description: longstring, group_class: "project"
     g = Group.find_by_uuid g.uuid
     assert_equal g.description, longstring
   end
@@ -248,7 +248,7 @@ class ArvadosModelTest < ActiveSupport::TestCase
 
   test 'create and retrieve using created_at time' do
     set_user_from_auth :active
-    group = Group.create! name: 'test create and retrieve group'
+    group = Group.create! name: 'test create and retrieve group', group_class: "project"
     assert group.valid?, "group is not valid"
 
     results = Group.where(created_at: group.created_at)
@@ -258,7 +258,7 @@ class ArvadosModelTest < ActiveSupport::TestCase
 
   test 'create and update twice and expect different update times' do
     set_user_from_auth :active
-    group = Group.create! name: 'test create and retrieve group'
+    group = Group.create! name: 'test create and retrieve group', group_class: "project"
     assert group.valid?, "group is not valid"
 
     # update 1
diff --git a/services/api/test/unit/collection_test.rb b/services/api/test/unit/collection_test.rb
index bf1ba517eb..addea83062 100644
--- a/services/api/test/unit/collection_test.rb
+++ b/services/api/test/unit/collection_test.rb
@@ -1000,6 +1000,19 @@ class CollectionTest < ActiveSupport::TestCase
   test "delete referring links in SweepTrashedObjects" do
     uuid = collections(:trashed_on_next_sweep).uuid
     act_as_system_user do
+      assert_raises ActiveRecord::RecordInvalid do
+        # Cannot create because :trashed_on_next_sweep is already trashed
+        Link.create!(head_uuid: uuid,
+                     tail_uuid: system_user_uuid,
+                     link_class: 'whatever',
+                     name: 'something')
+      end
+
+      # Bump trash_at to now + 1 minute
+      Collection.where(uuid: uuid).
+        update(trash_at: db_current_time + (1).minute)
+
+      # Not considered trashed now
       Link.create!(head_uuid: uuid,
                    tail_uuid: system_user_uuid,
                    link_class: 'whatever',
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 5f17efc445..98e60e0579 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -663,6 +663,8 @@ class ContainerTest < ActiveSupport::TestCase
 
     auth_exp = ApiClientAuthorization.find_by_uuid(auth_uuid_was).expires_at
     assert_operator auth_exp, :<, db_current_time
+
+    assert_nil ApiClientAuthorization.validate(token: ApiClientAuthorization.find_by_uuid(auth_uuid_was).token)
   end
 
   test "Exceed maximum lock-unlock cycles" do
diff --git a/services/api/test/unit/group_test.rb b/services/api/test/unit/group_test.rb
index 24d7333ab5..30fddfa5b8 100644
--- a/services/api/test/unit/group_test.rb
+++ b/services/api/test/unit/group_test.rb
@@ -3,8 +3,10 @@
 # SPDX-License-Identifier: AGPL-3.0
 
 require 'test_helper'
+require 'fix_roles_projects'
 
 class GroupTest < ActiveSupport::TestCase
+  include DbCurrentTime
 
   test "cannot set owner_uuid to object with existing ownership cycle" do
     set_user_from_auth :active_trustedclient
@@ -31,8 +33,8 @@ class GroupTest < ActiveSupport::TestCase
   test "cannot create a new ownership cycle" do
     set_user_from_auth :active_trustedclient
 
-    g_foo = Group.create!(name: "foo")
-    g_bar = Group.create!(name: "bar")
+    g_foo = Group.create!(name: "foo", group_class: "project")
+    g_bar = Group.create!(name: "bar", group_class: "project")
 
     g_foo.owner_uuid = g_bar.uuid
     assert g_foo.save, lambda { g_foo.errors.messages }
@@ -45,7 +47,7 @@ class GroupTest < ActiveSupport::TestCase
   test "cannot create a single-object ownership cycle" do
     set_user_from_auth :active_trustedclient
 
-    g_foo = Group.create!(name: "foo")
+    g_foo = Group.create!(name: "foo", group_class: "project")
     assert g_foo.save
 
     # Ensure I have permission to manage this group even when its owner changes
@@ -60,10 +62,47 @@ class GroupTest < ActiveSupport::TestCase
     assert g_foo.errors.messages[:owner_uuid].join(" ").match(/ownership cycle/)
   end
 
+  test "cannot create a group that is not a 'role' or 'project'" do
+    set_user_from_auth :active_trustedclient
+
+    assert_raises(ActiveRecord::RecordInvalid) do
+      Group.create!(name: "foo")
+    end
+
+    assert_raises(ActiveRecord::RecordInvalid) do
+      Group.create!(name: "foo", group_class: "")
+    end
+
+    assert_raises(ActiveRecord::RecordInvalid) do
+      Group.create!(name: "foo", group_class: "bogus")
+    end
+  end
+
+  test "cannot change group_class on an already created group" do
+    set_user_from_auth :active_trustedclient
+    g = Group.create!(name: "foo", group_class: "role")
+    assert_raises(ActiveRecord::RecordInvalid) do
+      g.update_attributes!(group_class: "project")
+    end
+  end
+
+  test "role cannot own things" do
+    set_user_from_auth :active_trustedclient
+    role = Group.create!(name: "foo", group_class: "role")
+    assert_raises(ArvadosModel::PermissionDeniedError) do
+      Collection.create!(name: "bzzz123", owner_uuid: role.uuid)
+    end
+
+    c = Collection.create!(name: "bzzz124")
+    assert_raises(ArvadosModel::PermissionDeniedError) do
+      c.update_attributes!(owner_uuid: role.uuid)
+    end
+  end
+
   test "trash group hides contents" do
     set_user_from_auth :active_trustedclient
 
-    g_foo = Group.create!(name: "foo")
+    g_foo = Group.create!(name: "foo", group_class: "project")
     col = Collection.create!(owner_uuid: g_foo.uuid)
 
     assert Collection.readable_by(users(:active)).where(uuid: col.uuid).any?
@@ -77,9 +116,9 @@ class GroupTest < ActiveSupport::TestCase
   test "trash group" do
     set_user_from_auth :active_trustedclient
 
-    g_foo = Group.create!(name: "foo")
-    g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
-    g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+    g_foo = Group.create!(name: "foo", group_class: "project")
+    g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+    g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
 
     assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
     assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
@@ -98,9 +137,9 @@ class GroupTest < ActiveSupport::TestCase
   test "trash subgroup" do
     set_user_from_auth :active_trustedclient
 
-    g_foo = Group.create!(name: "foo")
-    g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
-    g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+    g_foo = Group.create!(name: "foo", group_class: "project")
+    g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+    g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
 
     assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
     assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
@@ -118,9 +157,9 @@ class GroupTest < ActiveSupport::TestCase
   test "trash subsubgroup" do
     set_user_from_auth :active_trustedclient
 
-    g_foo = Group.create!(name: "foo")
-    g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid)
-    g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid)
+    g_foo = Group.create!(name: "foo", group_class: "project")
+    g_bar = Group.create!(name: "bar", owner_uuid: g_foo.uuid, group_class: "project")
+    g_baz = Group.create!(name: "baz", owner_uuid: g_bar.uuid, group_class: "project")
 
     assert Group.readable_by(users(:active)).where(uuid: g_foo.uuid).any?
     assert Group.readable_by(users(:active)).where(uuid: g_bar.uuid).any?
@@ -168,7 +207,7 @@ class GroupTest < ActiveSupport::TestCase
   test "trashed does not propagate across permission links" do
     set_user_from_auth :admin
 
-    g_foo = Group.create!(name: "foo")
+    g_foo = Group.create!(name: "foo", group_class: "role")
     u_bar = User.create!(first_name: "bar")
 
     assert Group.readable_by(users(:admin)).where(uuid: g_foo.uuid).any?
@@ -237,7 +276,8 @@ class GroupTest < ActiveSupport::TestCase
     set_user_from_auth :active
     ["", "{SOLIDUS}"].each do |subst|
       Rails.configuration.Collections.ForwardSlashNameSubstitution = subst
-      g = Group.create
+      proj = Group.create group_class: "project"
+      role = Group.create group_class: "role"
       [[nil, true],
        ["", true],
        [".", false],
@@ -248,12 +288,70 @@ class GroupTest < ActiveSupport::TestCase
        ["../..", subst != ""],
        ["/", subst != ""],
       ].each do |name, valid|
-        g.name = name
-        g.group_class = "role"
-        assert_equal true, g.valid?
-        g.group_class = "project"
-        assert_equal valid, g.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}"
+        role.name = name
+        assert_equal true, role.valid?
+        proj.name = name
+        assert_equal valid, proj.valid?, "#{name.inspect} should be #{valid ? "valid" : "invalid"}"
       end
     end
   end
+
+  def insert_group uuid, owner_uuid, name, group_class
+    q = ActiveRecord::Base.connection.exec_query %{
+insert into groups (uuid, owner_uuid, name, group_class, created_at, updated_at)
+       values ('#{uuid}', '#{owner_uuid}',
+               '#{name}', #{if group_class then "'"+group_class+"'" else 'NULL' end},
+               statement_timestamp(), statement_timestamp())
+}
+    uuid
+  end
+
+  test "migration to fix roles and projects" do
+    g1 = insert_group Group.generate_uuid, system_user_uuid, 'group with no class', nil
+    g2 = insert_group Group.generate_uuid, users(:active).uuid, 'role owned by a user', 'role'
+
+    g3 = insert_group Group.generate_uuid, system_user_uuid, 'role that owns a project', 'role'
+    g4 = insert_group Group.generate_uuid, g3, 'the project', 'project'
+
+    g5 = insert_group Group.generate_uuid, users(:active).uuid, 'a project with an outgoing permission link', 'project'
+
+    g6 = insert_group Group.generate_uuid, system_user_uuid, 'name collision', 'role'
+    g7 = insert_group Group.generate_uuid, users(:active).uuid, 'name collision', 'role'
+
+    g8 = insert_group Group.generate_uuid, users(:active).uuid, 'trashed with no class', nil
+    g8obj = Group.find_by_uuid(g8)
+    g8obj.trash_at = db_current_time
+    g8obj.delete_at = db_current_time
+    act_as_system_user do
+      g8obj.save!(validate: false)
+    end
+
+    refresh_permissions
+
+    act_as_system_user do
+      l1 = Link.create!(link_class: 'permission', name: 'can_manage', tail_uuid: g3, head_uuid: g4)
+      q = ActiveRecord::Base.connection.exec_query %{
+update links set tail_uuid='#{g5}' where uuid='#{l1.uuid}'
+}
+    refresh_permissions
+    end
+
+    assert_equal nil, Group.find_by_uuid(g1).group_class
+    assert_equal nil, Group.find_by_uuid(g8).group_class
+    assert_equal users(:active).uuid, Group.find_by_uuid(g2).owner_uuid
+    assert_equal g3, Group.find_by_uuid(g4).owner_uuid
+    assert !Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
+    assert !Link.where(tail_uuid: g3, head_uuid: g4, link_class: "permission", name: "can_manage").any?
+    assert Link.where(link_class: 'permission', name: 'can_manage', tail_uuid: g5, head_uuid: g4).any?
+
+    fix_roles_projects
+
+    assert_equal 'role', Group.find_by_uuid(g1).group_class
+    assert_equal 'role', Group.find_by_uuid(g8).group_class
+    assert_equal system_user_uuid, Group.find_by_uuid(g2).owner_uuid
+    assert_equal system_user_uuid, Group.find_by_uuid(g4).owner_uuid
+    assert Link.where(tail_uuid: users(:active).uuid, head_uuid: g2, link_class: "permission", name: "can_manage").any?
+    assert Link.where(tail_uuid: g3, head_uuid: g4, link_class: "permission", name: "can_manage").any?
+    assert !Link.where(link_class: 'permission', name: 'can_manage', tail_uuid: g5, head_uuid: g4).any?
+  end
 end
diff --git a/services/api/test/unit/owner_test.rb b/services/api/test/unit/owner_test.rb
index 528c6d253f..e356f4d9fa 100644
--- a/services/api/test/unit/owner_test.rb
+++ b/services/api/test/unit/owner_test.rb
@@ -21,7 +21,11 @@ class OwnerTest < ActiveSupport::TestCase
   Group.all
   [User, Group].each do |o_class|
     test "create object with legit #{o_class} owner" do
-      o = o_class.create!
+      if o_class == Group
+        o = o_class.create! group_class: "project"
+      else
+        o = o_class.create!
+      end
       i = Specimen.create(owner_uuid: o.uuid)
       assert i.valid?, "new item should pass validation"
       assert i.uuid, "new item should have an ID"
@@ -44,9 +48,19 @@ class OwnerTest < ActiveSupport::TestCase
 
     [User, Group].each do |new_o_class|
       test "change owner from legit #{o_class} to legit #{new_o_class} owner" do
-        o = o_class.create!
+        o = if o_class == Group
+              o_class.create! group_class: "project"
+            else
+              o_class.create!
+            end
         i = Specimen.create!(owner_uuid: o.uuid)
-        new_o = new_o_class.create!
+
+        new_o = if new_o_class == Group
+              new_o_class.create! group_class: "project"
+            else
+              new_o_class.create!
+            end
+
         assert(Specimen.where(uuid: i.uuid).any?,
                "new item should really be in DB")
         assert(i.update_attributes(owner_uuid: new_o.uuid),
@@ -55,7 +69,11 @@ class OwnerTest < ActiveSupport::TestCase
     end
 
     test "delete #{o_class} that owns nothing" do
-      o = o_class.create!
+      if o_class == Group
+        o = o_class.create! group_class: "project"
+      else
+        o = o_class.create!
+      end
       assert(o_class.where(uuid: o.uuid).any?,
              "new #{o_class} should really be in DB")
       assert(o.destroy, "should delete #{o_class} that owns nothing")
@@ -65,13 +83,21 @@ class OwnerTest < ActiveSupport::TestCase
 
     test "change uuid of #{o_class} that owns nothing" do
       # (we're relying on our admin credentials here)
-      o = o_class.create!
+      if o_class == Group
+        o = o_class.create! group_class: "project"
+      else
+        o = o_class.create!
+      end
       assert(o_class.where(uuid: o.uuid).any?,
              "new #{o_class} should really be in DB")
       old_uuid = o.uuid
       new_uuid = o.uuid.sub(/..........$/, rand(2**256).to_s(36)[0..9])
-      assert(o.update_attributes(uuid: new_uuid),
-             "should change #{o_class} uuid from #{old_uuid} to #{new_uuid}")
+      if o.respond_to? :update_uuid
+        o.update_uuid(new_uuid: new_uuid)
+      else
+        assert(o.update_attributes(uuid: new_uuid),
+               "should change #{o_class} uuid from #{old_uuid} to #{new_uuid}")
+      end
       assert_equal(false, o_class.where(uuid: old_uuid).any?,
                    "#{old_uuid} should disappear when renamed to #{new_uuid}")
     end
@@ -83,9 +109,11 @@ class OwnerTest < ActiveSupport::TestCase
       assert_equal(true, Specimen.where(owner_uuid: o.uuid).any?,
                    "need something to be owned by #{o.uuid} for this test")
 
-      assert_raises(ActiveRecord::DeleteRestrictionError,
-                    "should not delete #{ofixt} that owns objects") do
-        o.destroy
+      skip_check_permissions_against_full_refresh do
+        assert_raises(ActiveRecord::DeleteRestrictionError,
+                      "should not delete #{ofixt} that owns objects") do
+          o.destroy
+        end
       end
     end
 
@@ -104,9 +132,14 @@ class OwnerTest < ActiveSupport::TestCase
     assert User.where(uuid: o.uuid).any?, "new User should really be in DB"
     assert_equal(true, o.update_attributes(owner_uuid: o.uuid),
                  "setting owner to self should work")
-    assert(o.destroy, "should delete User that owns self")
+
+    skip_check_permissions_against_full_refresh do
+      assert(o.destroy, "should delete User that owns self")
+    end
+
     assert_equal(false, User.where(uuid: o.uuid).any?,
                  "#{o.uuid} should not be in DB after deleting")
+    check_permissions_against_full_refresh
   end
 
   test "change uuid of User that owns self" do
@@ -116,8 +149,8 @@ class OwnerTest < ActiveSupport::TestCase
                  "setting owner to self should work")
     old_uuid = o.uuid
     new_uuid = o.uuid.sub(/..........$/, rand(2**256).to_s(36)[0..9])
-    assert(o.update_attributes(uuid: new_uuid),
-           "should change uuid of User that owns self")
+    o.update_uuid(new_uuid: new_uuid)
+    o = User.find_by_uuid(new_uuid)
     assert_equal(false, User.where(uuid: old_uuid).any?,
                  "#{old_uuid} should not be in DB after deleting")
     assert_equal(true, User.where(uuid: new_uuid).any?,
diff --git a/services/api/test/unit/permission_test.rb b/services/api/test/unit/permission_test.rb
index 18d2fbbcb5..10664474c6 100644
--- a/services/api/test/unit/permission_test.rb
+++ b/services/api/test/unit/permission_test.rb
@@ -10,7 +10,7 @@ class PermissionTest < ActiveSupport::TestCase
   test "Grant permissions on an object I own" do
     set_user_from_auth :active_trustedclient
 
-    ob = Specimen.create
+    ob = Collection.create
     assert ob.save
 
     # Ensure I have permission to manage this group even when its owner changes
@@ -24,7 +24,7 @@ class PermissionTest < ActiveSupport::TestCase
   test "Delete permission links when deleting an object" do
     set_user_from_auth :active_trustedclient
 
-    ob = Specimen.create!
+    ob = Collection.create!
     Link.create!(tail_uuid: users(:active).uuid,
                  head_uuid: ob.uuid,
                  link_class: 'permission',
@@ -37,7 +37,7 @@ class PermissionTest < ActiveSupport::TestCase
 
   test "permission links owned by root" do
     set_user_from_auth :active_trustedclient
-    ob = Specimen.create!
+    ob = Collection.create!
     perm_link = Link.create!(tail_uuid: users(:active).uuid,
                              head_uuid: ob.uuid,
                              link_class: 'permission',
@@ -46,20 +46,20 @@ class PermissionTest < ActiveSupport::TestCase
   end
 
   test "readable_by" do
-    set_user_from_auth :active_trustedclient
+    set_user_from_auth :admin
 
-    ob = Specimen.create!
+    ob = Collection.create!
     Link.create!(tail_uuid: users(:active).uuid,
                  head_uuid: ob.uuid,
                  link_class: 'permission',
                  name: 'can_read')
-    assert Specimen.readable_by(users(:active)).where(uuid: ob.uuid).any?, "user does not have read permission"
+    assert Collection.readable_by(users(:active)).where(uuid: ob.uuid).any?, "user does not have read permission"
   end
 
   test "writable_by" do
-    set_user_from_auth :active_trustedclient
+    set_user_from_auth :admin
 
-    ob = Specimen.create!
+    ob = Collection.create!
     Link.create!(tail_uuid: users(:active).uuid,
                  head_uuid: ob.uuid,
                  link_class: 'permission',
@@ -67,6 +67,34 @@ class PermissionTest < ActiveSupport::TestCase
     assert ob.writable_by.include?(users(:active).uuid), "user does not have write permission"
   end
 
+  test "update permission link" do
+    set_user_from_auth :admin
+
+    grp = Group.create! name: "blah project", group_class: "project"
+    ob = Collection.create! owner_uuid: grp.uuid
+
+    assert !users(:active).can?(write: ob)
+    assert !users(:active).can?(read: ob)
+
+    l1 = Link.create!(tail_uuid: users(:active).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_write')
+
+    assert users(:active).can?(write: ob)
+    assert users(:active).can?(read: ob)
+
+    l1.update_attributes!(name: 'can_read')
+
+    assert !users(:active).can?(write: ob)
+    assert users(:active).can?(read: ob)
+
+    l1.destroy
+
+    assert !users(:active).can?(write: ob)
+    assert !users(:active).can?(read: ob)
+  end
+
   test "writable_by reports requesting user's own uuid for a writable project" do
     invited_to_write = users(:project_viewer)
     group = groups(:asubproject)
@@ -124,16 +152,17 @@ class PermissionTest < ActiveSupport::TestCase
   test "user owns group, group can_manage object's group, user can add permissions" do
     set_user_from_auth :admin
 
-    owner_grp = Group.create!(owner_uuid: users(:active).uuid)
+    owner_grp = Group.create!(owner_uuid: users(:active).uuid, group_class: "role")
 
-    sp_grp = Group.create!
-    sp = Specimen.create!(owner_uuid: sp_grp.uuid)
+    sp_grp = Group.create!(group_class: "project")
 
     Link.create!(link_class: 'permission',
                  name: 'can_manage',
                  tail_uuid: owner_grp.uuid,
                  head_uuid: sp_grp.uuid)
 
+    sp = Collection.create!(owner_uuid: sp_grp.uuid)
+
     # active user owns owner_grp, which has can_manage permission on sp_grp
     # user should be able to add permissions on sp.
     set_user_from_auth :active_trustedclient
@@ -149,7 +178,7 @@ class PermissionTest < ActiveSupport::TestCase
   skip "can_manage permission on a non-group object" do
     set_user_from_auth :admin
 
-    ob = Specimen.create!
+    ob = Collection.create!
     # grant can_manage permission to active
     perm_link = Link.create!(tail_uuid: users(:active).uuid,
                              head_uuid: ob.uuid,
@@ -170,7 +199,7 @@ class PermissionTest < ActiveSupport::TestCase
   test "user without can_manage permission may not modify permission link" do
     set_user_from_auth :admin
 
-    ob = Specimen.create!
+    ob = Collection.create!
     # grant can_manage permission to active
     perm_link = Link.create!(tail_uuid: users(:active).uuid,
                              head_uuid: ob.uuid,
@@ -192,13 +221,14 @@ class PermissionTest < ActiveSupport::TestCase
     manager = create :active_user, first_name: "Manage", last_name: "Er"
     minion = create :active_user, first_name: "Min", last_name: "Ion"
     minions_specimen = act_as_user minion do
-      Specimen.create!
+      g = Group.create! name: "minon project", group_class: "project"
+      Collection.create! owner_uuid: g.uuid
     end
     # Manager creates a group. (Make sure it doesn't magically give
     # anyone any additional permissions.)
     g = nil
     act_as_user manager do
-      g = create :group, name: "NoBigSecret Lab"
+      g = create :group, name: "NoBigSecret Lab", group_class: "role"
       assert_empty(User.readable_by(manager).where(uuid: minion.uuid),
                    "saw a user I shouldn't see")
       assert_raises(ArvadosModel::PermissionDeniedError,
@@ -255,7 +285,7 @@ class PermissionTest < ActiveSupport::TestCase
         create(:permission_link,
                name: 'can_manage', tail_uuid: manager.uuid, head_uuid: minion.uuid)
       end
-      assert_empty(Specimen
+      assert_empty(Collection
                      .readable_by(manager)
                      .where(uuid: minions_specimen.uuid),
                    "manager saw the minion's private stuff")
@@ -273,7 +303,7 @@ class PermissionTest < ActiveSupport::TestCase
 
     act_as_user manager do
       # Now, manager can read and write Minion's stuff.
-      assert_not_empty(Specimen
+      assert_not_empty(Collection
                          .readable_by(manager)
                          .where(uuid: minions_specimen.uuid),
                        "manager could not find minion's specimen by uuid")
@@ -294,7 +324,7 @@ class PermissionTest < ActiveSupport::TestCase
                      "#{a.first_name} should not be able to see 'b' in the user list")
 
     act_as_system_user do
-      g = create :group
+      g = create :group, group_class: "role"
       [a,b].each do |u|
         create(:permission_link,
                name: 'can_read', tail_uuid: u.uuid, head_uuid: g.uuid)
@@ -309,12 +339,12 @@ class PermissionTest < ActiveSupport::TestCase
                      "#{a.first_name} should be able to see 'b' in the user list")
 
     a_specimen = act_as_user a do
-      Specimen.create!
+      Collection.create!
     end
-    assert_not_empty(Specimen.readable_by(a).where(uuid: a_specimen.uuid),
-                     "A cannot read own Specimen, following test probably useless.")
-    assert_empty(Specimen.readable_by(b).where(uuid: a_specimen.uuid),
-                 "B can read A's Specimen")
+    assert_not_empty(Collection.readable_by(a).where(uuid: a_specimen.uuid),
+                     "A cannot read own Collection, following test probably useless.")
+    assert_empty(Collection.readable_by(b).where(uuid: a_specimen.uuid),
+                 "B can read A's Collection")
     [a,b].each do |u|
       assert_empty(User.readable_by(u).where(uuid: other.uuid),
                    "#{u.first_name} can see OTHER in the user list")
@@ -341,13 +371,13 @@ class PermissionTest < ActiveSupport::TestCase
   test "cannot create with owner = unwritable user" do
     set_user_from_auth :rominiadmin
     assert_raises ArvadosModel::PermissionDeniedError, "created with owner = unwritable user" do
-      Specimen.create!(owner_uuid: users(:active).uuid)
+      Collection.create!(owner_uuid: users(:active).uuid)
     end
   end
 
   test "cannot change owner to unwritable user" do
     set_user_from_auth :rominiadmin
-    ob = Specimen.create!
+    ob = Collection.create!
     assert_raises ArvadosModel::PermissionDeniedError, "changed owner to unwritable user" do
       ob.update_attributes!(owner_uuid: users(:active).uuid)
     end
@@ -356,13 +386,13 @@ class PermissionTest < ActiveSupport::TestCase
   test "cannot create with owner = unwritable group" do
     set_user_from_auth :rominiadmin
     assert_raises ArvadosModel::PermissionDeniedError, "created with owner = unwritable group" do
-      Specimen.create!(owner_uuid: groups(:aproject).uuid)
+      Collection.create!(owner_uuid: groups(:aproject).uuid)
     end
   end
 
   test "cannot change owner to unwritable group" do
     set_user_from_auth :rominiadmin
-    ob = Specimen.create!
+    ob = Collection.create!
     assert_raises ArvadosModel::PermissionDeniedError, "changed owner to unwritable group" do
       ob.update_attributes!(owner_uuid: groups(:aproject).uuid)
     end
@@ -390,4 +420,163 @@ class PermissionTest < ActiveSupport::TestCase
 
     assert_not_empty container_logs(:running_older, :anonymous)
   end
+
+  test "add user to group, then remove them" do
+    set_user_from_auth :admin
+    grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+    col = Collection.create!(owner_uuid: system_user_uuid)
+
+    l0 = Link.create!(tail_uuid: grp.uuid,
+                 head_uuid: col.uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+
+    assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+    assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+    l1 = Link.create!(tail_uuid: users(:active).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+    l2 = Link.create!(tail_uuid: grp.uuid,
+                 head_uuid: users(:active).uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+
+    l3 = Link.create!(tail_uuid: users(:project_viewer).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+    l4 = Link.create!(tail_uuid: grp.uuid,
+                 head_uuid: users(:project_viewer).uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid).first
+
+    l1.destroy
+    l2.destroy
+
+    assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+    assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+  end
+
+
+  test "add user to group, then change permission level" do
+    set_user_from_auth :admin
+    grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+    col = Collection.create!(owner_uuid: grp.uuid)
+    assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+    assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+    l1 = Link.create!(tail_uuid: users(:active).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_manage')
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert users(:active).can?(read: col.uuid)
+    assert users(:active).can?(write: col.uuid)
+    assert users(:active).can?(manage: col.uuid)
+
+    l1.name = 'can_read'
+    l1.save!
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert users(:active).can?(read: col.uuid)
+    assert !users(:active).can?(write: col.uuid)
+    assert !users(:active).can?(manage: col.uuid)
+
+    l1.name = 'can_write'
+    l1.save!
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert users(:active).can?(read: col.uuid)
+    assert users(:active).can?(write: col.uuid)
+    assert !users(:active).can?(manage: col.uuid)
+  end
+
+
+  test "add user to group, then add overlapping permission link to group" do
+    set_user_from_auth :admin
+    grp = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+    col = Collection.create!(owner_uuid: grp.uuid)
+    assert_empty Collection.readable_by(users(:active)).where(uuid: col.uuid)
+    assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+    l1 = Link.create!(tail_uuid: users(:active).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_manage')
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert users(:active).can?(read: col.uuid)
+    assert users(:active).can?(write: col.uuid)
+    assert users(:active).can?(manage: col.uuid)
+
+    l3 = Link.create!(tail_uuid: users(:active).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert users(:active).can?(read: col.uuid)
+    assert users(:active).can?(write: col.uuid)
+    assert users(:active).can?(manage: col.uuid)
+
+    l3.destroy!
+
+    assert Collection.readable_by(users(:active)).where(uuid: col.uuid).first
+    assert users(:active).can?(read: col.uuid)
+    assert users(:active).can?(write: col.uuid)
+    assert users(:active).can?(manage: col.uuid)
+  end
+
+
+  test "add user to group, then add overlapping permission link to subproject" do
+    set_user_from_auth :admin
+    grp = Group.create!(owner_uuid: system_user_uuid, group_class: "role")
+    prj = Group.create!(owner_uuid: system_user_uuid, group_class: "project")
+
+    l0 = Link.create!(tail_uuid: grp.uuid,
+                 head_uuid: prj.uuid,
+                 link_class: 'permission',
+                 name: 'can_manage')
+
+    assert_empty Group.readable_by(users(:active)).where(uuid: prj.uuid)
+    assert_empty User.readable_by(users(:active)).where(uuid: users(:project_viewer).uuid)
+
+    l1 = Link.create!(tail_uuid: users(:active).uuid,
+                 head_uuid: grp.uuid,
+                 link_class: 'permission',
+                 name: 'can_manage')
+    l2 = Link.create!(tail_uuid: grp.uuid,
+                 head_uuid: users(:active).uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+
+    assert Group.readable_by(users(:active)).where(uuid: prj.uuid).first
+    assert users(:active).can?(read: prj.uuid)
+    assert users(:active).can?(write: prj.uuid)
+    assert users(:active).can?(manage: prj.uuid)
+
+    l3 = Link.create!(tail_uuid: grp.uuid,
+                 head_uuid: prj.uuid,
+                 link_class: 'permission',
+                 name: 'can_read')
+
+    assert Group.readable_by(users(:active)).where(uuid: prj.uuid).first
+    assert users(:active).can?(read: prj.uuid)
+    assert users(:active).can?(write: prj.uuid)
+    assert users(:active).can?(manage: prj.uuid)
+
+    l3.destroy!
+
+    assert Group.readable_by(users(:active)).where(uuid: prj.uuid).first
+    assert users(:active).can?(read: prj.uuid)
+    assert users(:active).can?(write: prj.uuid)
+    assert users(:active).can?(manage: prj.uuid)
+  end
 end
diff --git a/services/api/test/unit/time_zone_test.rb b/services/api/test/unit/time_zone_test.rb
new file mode 100644
index 0000000000..60ca6b50b6
--- /dev/null
+++ b/services/api/test/unit/time_zone_test.rb
@@ -0,0 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'test_helper'
+
+class TimeZoneTest < ActiveSupport::TestCase
+  test "Database connection time zone" do
+    # This is pointless if the testing host is already using the UTC
+    # time zone.  But if not, the test confirms that
+    # config/initializers/time_zone.rb has successfully changed the
+    # database connection time zone to UTC.
+    assert_equal('UTC', ActiveRecord::Base.connection.select_value("show timezone"))
+  end
+end
diff --git a/services/api/test/unit/user_test.rb b/services/api/test/unit/user_test.rb
index 260795c12f..7fcd36d709 100644
--- a/services/api/test/unit/user_test.rb
+++ b/services/api/test/unit/user_test.rb
@@ -165,7 +165,9 @@ class UserTest < ActiveSupport::TestCase
 
       if auto_admin_first_user_config
         # This test requires no admin users exist (except for the system user)
-        users(:admin).delete
+        act_as_system_user do
+          users(:admin).update_attributes!(is_admin: false)
+        end
         @all_users = User.where("uuid not like '%-000000000000000'").where(:is_admin => true)
         assert_equal 0, @all_users.count, "No admin users should exist (except for the system user)"
       end
@@ -476,15 +478,6 @@ class UserTest < ActiveSupport::TestCase
 
     vm = VirtualMachine.create
 
-    # Set up the bogus Link
-    bad_uuid = 'zzzzz-tpzed-xyzxyzxyzxyzxyz'
-
-    resp_link = Link.create ({tail_uuid: email, link_class: 'permission',
-        name: 'can_login', head_uuid: bad_uuid})
-    resp_link.save(validate: false)
-
-    verify_link resp_link, 'permission', 'can_login', email, bad_uuid
-
     response = user.setup(repo_name: 'foo/testrepo',
                           vm_uuid: vm.uuid)
 
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 36bcef4f26..4115482d80 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -280,7 +280,8 @@ func (disp *Dispatcher) runContainer(_ *dispatch.Dispatcher, ctr arvados.Contain
 		cmd = append(cmd, disp.cluster.Containers.CrunchRunArgumentsList...)
 		if err := disp.submit(ctr, cmd); err != nil {
 			var text string
-			if err, ok := err.(dispatchcloud.ConstraintsNotSatisfiableError); ok {
+			switch err := err.(type) {
+			case dispatchcloud.ConstraintsNotSatisfiableError:
 				var logBuf bytes.Buffer
 				fmt.Fprintf(&logBuf, "cannot run container %s: %s\n", ctr.UUID, err)
 				if len(err.AvailableTypes) == 0 {
@@ -296,7 +297,7 @@ func (disp *Dispatcher) runContainer(_ *dispatch.Dispatcher, ctr arvados.Contain
 				}
 				text = logBuf.String()
 				disp.UpdateState(ctr.UUID, dispatch.Cancelled)
-			} else {
+			default:
 				text = fmt.Sprintf("Error submitting container %s to slurm: %s", ctr.UUID, err)
 			}
 			log.Print(text)
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 3c14f7044c..480434de65 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -424,7 +424,7 @@ BatchSize: 99
 	err = s.disp.configure("crunch-dispatch-slurm", []string{"-config", tmpfile.Name()})
 	c.Check(err, IsNil)
 
-	c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(s.disp.cluster.Services.Controller.ExternalURL, Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 	c.Check(s.disp.cluster.SystemRootToken, Equals, "abcdefg")
 	c.Check(s.disp.cluster.Containers.SLURM.SbatchArgumentsList, DeepEquals, []string{"--foo", "bar"})
 	c.Check(s.disp.cluster.Containers.CloudVMs.PollInterval, Equals, arvados.Duration(12*time.Second))
diff --git a/services/fuse/fpm-info.sh b/services/fuse/fpm-info.sh
index fd94ef7afa..f789abe692 100644
--- a/services/fuse/fpm-info.sh
+++ b/services/fuse/fpm-info.sh
@@ -9,6 +9,6 @@ case "$TARGET" in
         fpm_depends+=(fuse-libs)
         ;;
     debian* | ubuntu*)
-        fpm_depends+=(libcurl3-gnutls libpython2.7)
+        fpm_depends+=(libcurl3-gnutls)
         ;;
 esac
diff --git a/services/fuse/tests/test_mount.py b/services/fuse/tests/test_mount.py
index 593d945cff..b2816ac16f 100644
--- a/services/fuse/tests/test_mount.py
+++ b/services/fuse/tests/test_mount.py
@@ -128,7 +128,7 @@ class FuseMagicTest(MountTestBase):
         super(FuseMagicTest, self).setUp(api=api)
 
         self.test_project = run_test_server.fixture('groups')['aproject']['uuid']
-        self.non_project_group = run_test_server.fixture('groups')['public']['uuid']
+        self.non_project_group = run_test_server.fixture('groups')['public_role']['uuid']
         self.collection_in_test_project = run_test_server.fixture('collections')['foo_collection_in_aproject']['name']
 
         cw = arvados.CollectionWriter()
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 3c35d304cb..86423a2976 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -6,6 +6,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"crypto/md5"
 	"fmt"
 	"io"
@@ -71,6 +72,9 @@ func (bal *Balancer) Run(client *arvados.Client, cluster *arvados.Cluster, runOp
 
 	defer bal.time("sweep", "wall clock time to run one full sweep")()
 
+	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(cluster.Collections.BalanceTimeout.Duration()))
+	defer cancel()
+
 	var lbFile *os.File
 	if bal.LostBlocksFile != "" {
 		tmpfn := bal.LostBlocksFile + ".tmp"
@@ -111,13 +115,21 @@ func (bal *Balancer) Run(client *arvados.Client, cluster *arvados.Cluster, runOp
 	if err = bal.CheckSanityEarly(client); err != nil {
 		return
 	}
+
+	// On a big site, indexing and sending trash/pull lists can
+	// take much longer than the usual 5 minute client
+	// timeout. From here on, we rely on the context deadline
+	// instead, aborting the entire operation if any part takes
+	// too long.
+	client.Timeout = 0
+
 	rs := bal.rendezvousState()
 	if runOptions.CommitTrash && rs != runOptions.SafeRendezvousState {
 		if runOptions.SafeRendezvousState != "" {
 			bal.logf("notice: KeepServices list has changed since last run")
 		}
 		bal.logf("clearing existing trash lists, in case the new rendezvous order differs from previous run")
-		if err = bal.ClearTrashLists(client); err != nil {
+		if err = bal.ClearTrashLists(ctx, client); err != nil {
 			return
 		}
 		// The current rendezvous state becomes "safe" (i.e.,
@@ -126,7 +138,8 @@ func (bal *Balancer) Run(client *arvados.Client, cluster *arvados.Cluster, runOp
 		// succeed in clearing existing trash lists.
 		nextRunOptions.SafeRendezvousState = rs
 	}
-	if err = bal.GetCurrentState(client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
+
+	if err = bal.GetCurrentState(ctx, client, cluster.Collections.BalanceCollectionBatch, cluster.Collections.BalanceCollectionBuffers); err != nil {
 		return
 	}
 	bal.ComputeChangeSets()
@@ -146,14 +159,14 @@ func (bal *Balancer) Run(client *arvados.Client, cluster *arvados.Cluster, runOp
 		lbFile = nil
 	}
 	if runOptions.CommitPulls {
-		err = bal.CommitPulls(client)
+		err = bal.CommitPulls(ctx, client)
 		if err != nil {
 			// Skip trash if we can't pull. (Too cautious?)
 			return
 		}
 	}
 	if runOptions.CommitTrash {
-		err = bal.CommitTrash(client)
+		err = bal.CommitTrash(ctx, client)
 	}
 	return
 }
@@ -286,11 +299,11 @@ func (bal *Balancer) rendezvousState() string {
 // We avoid this problem if we clear all trash lists before getting
 // indexes. (We also assume there is only one rebalancing process
 // running at a time.)
-func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
+func (bal *Balancer) ClearTrashLists(ctx context.Context, c *arvados.Client) error {
 	for _, srv := range bal.KeepServices {
 		srv.ChangeSet = &ChangeSet{}
 	}
-	return bal.CommitTrash(c)
+	return bal.CommitTrash(ctx, c)
 }
 
 // GetCurrentState determines the current replication state, and the
@@ -304,7 +317,10 @@ func (bal *Balancer) ClearTrashLists(c *arvados.Client) error {
 // collection manifests in the database (API server).
 //
 // It encodes the resulting information in BlockStateMap.
-func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) error {
+func (bal *Balancer) GetCurrentState(ctx context.Context, c *arvados.Client, pageSize, bufs int) error {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	defer bal.time("get_state", "wall clock time to get current state")()
 	bal.BlockStateMap = NewBlockStateMap()
 
@@ -348,12 +364,13 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
 		go func(mounts []*KeepMount) {
 			defer wg.Done()
 			bal.logf("mount %s: retrieve index from %s", mounts[0], mounts[0].KeepService)
-			idx, err := mounts[0].KeepService.IndexMount(c, mounts[0].UUID, "")
+			idx, err := mounts[0].KeepService.IndexMount(ctx, c, mounts[0].UUID, "")
 			if err != nil {
 				select {
 				case errs <- fmt.Errorf("%s: retrieve index: %v", mounts[0], err):
 				default:
 				}
+				cancel()
 				return
 			}
 			if len(errs) > 0 {
@@ -391,6 +408,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
 				}
 				for range collQ {
 				}
+				cancel()
 				return
 			}
 			bal.collScanned++
@@ -402,7 +420,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		err = EachCollection(c, pageSize,
+		err = EachCollection(ctx, c, pageSize,
 			func(coll arvados.Collection) error {
 				collQ <- coll
 				if len(errs) > 0 {
@@ -422,6 +440,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
 			case errs <- err:
 			default:
 			}
+			cancel()
 		}
 	}()
 
@@ -1084,22 +1103,22 @@ func (bal *Balancer) CheckSanityLate() error {
 // keepstore servers. This has the effect of increasing replication of
 // existing blocks that are either underreplicated or poorly
 // distributed according to rendezvous hashing.
-func (bal *Balancer) CommitPulls(c *arvados.Client) error {
+func (bal *Balancer) CommitPulls(ctx context.Context, c *arvados.Client) error {
 	defer bal.time("send_pull_lists", "wall clock time to send pull lists")()
 	return bal.commitAsync(c, "send pull list",
 		func(srv *KeepService) error {
-			return srv.CommitPulls(c)
+			return srv.CommitPulls(ctx, c)
 		})
 }
 
 // CommitTrash sends the computed lists of trash requests to the
 // keepstore servers. This has the effect of deleting blocks that are
 // overreplicated or unreferenced.
-func (bal *Balancer) CommitTrash(c *arvados.Client) error {
+func (bal *Balancer) CommitTrash(ctx context.Context, c *arvados.Client) error {
 	defer bal.time("send_trash_lists", "wall clock time to send trash lists")()
 	return bal.commitAsync(c, "send trash list",
 		func(srv *KeepService) error {
-			return srv.CommitTrash(c)
+			return srv.CommitTrash(ctx, c)
 		})
 }
 
diff --git a/services/keep-balance/collection.go b/services/keep-balance/collection.go
index c4ddc90c41..1659918caf 100644
--- a/services/keep-balance/collection.go
+++ b/services/keep-balance/collection.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"time"
 
@@ -30,7 +31,7 @@ func countCollections(c *arvados.Client, params arvados.ResourceListParams) (int
 //
 // If pageSize > 0 it is used as the maximum page size in each API
 // call; otherwise the maximum allowed page size is requested.
-func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
+func EachCollection(ctx context.Context, c *arvados.Client, pageSize int, f func(arvados.Collection) error, progress func(done, total int)) error {
 	if progress == nil {
 		progress = func(_, _ int) {}
 	}
@@ -75,7 +76,7 @@ func EachCollection(c *arvados.Client, pageSize int, f func(arvados.Collection)
 	for {
 		progress(callCount, expectCount)
 		var page arvados.CollectionList
-		err := c.RequestAndDecode(&page, "GET", "arvados/v1/collections", nil, params)
+		err := c.RequestAndDecodeContext(ctx, &page, "GET", "arvados/v1/collections", nil, params)
 		if err != nil {
 			return err
 		}
diff --git a/services/keep-balance/collection_test.go b/services/keep-balance/collection_test.go
index f8921c294a..3ab9d07b2e 100644
--- a/services/keep-balance/collection_test.go
+++ b/services/keep-balance/collection_test.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"context"
 	"sync"
 	"time"
 
@@ -29,7 +30,7 @@ func (s *integrationSuite) TestIdenticalTimestamps(c *check.C) {
 			longestStreak := 0
 			var lastMod time.Time
 			sawUUID := make(map[string]bool)
-			err := EachCollection(s.client, pageSize, func(c arvados.Collection) error {
+			err := EachCollection(context.Background(), s.client, pageSize, func(c arvados.Collection) error {
 				if c.ModifiedAt.IsZero() {
 					return nil
 				}
diff --git a/services/keep-balance/keep_service.go b/services/keep-balance/keep_service.go
index e2adf1a4b7..17f8418f62 100644
--- a/services/keep-balance/keep_service.go
+++ b/services/keep-balance/keep_service.go
@@ -5,6 +5,7 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -35,19 +36,19 @@ func (srv *KeepService) URLBase() string {
 
 // CommitPulls sends the current list of pull requests to the storage
 // server (even if the list is empty).
-func (srv *KeepService) CommitPulls(c *arvados.Client) error {
-	return srv.put(c, "pull", srv.ChangeSet.Pulls)
+func (srv *KeepService) CommitPulls(ctx context.Context, c *arvados.Client) error {
+	return srv.put(ctx, c, "pull", srv.ChangeSet.Pulls)
 }
 
 // CommitTrash sends the current list of trash requests to the storage
 // server (even if the list is empty).
-func (srv *KeepService) CommitTrash(c *arvados.Client) error {
-	return srv.put(c, "trash", srv.ChangeSet.Trashes)
+func (srv *KeepService) CommitTrash(ctx context.Context, c *arvados.Client) error {
+	return srv.put(ctx, c, "trash", srv.ChangeSet.Trashes)
 }
 
 // Perform a PUT request at path, with data (as JSON) in the request
 // body.
-func (srv *KeepService) put(c *arvados.Client, path string, data interface{}) error {
+func (srv *KeepService) put(ctx context.Context, c *arvados.Client, path string, data interface{}) error {
 	// We'll start a goroutine to do the JSON encoding, so we can
 	// stream it to the http client through a Pipe, rather than
 	// keeping the entire encoded version in memory.
@@ -64,7 +65,7 @@ func (srv *KeepService) put(c *arvados.Client, path string, data interface{}) er
 	}()
 
 	url := srv.URLBase() + "/" + path
-	req, err := http.NewRequest("PUT", url, ioutil.NopCloser(jsonR))
+	req, err := http.NewRequestWithContext(ctx, "PUT", url, ioutil.NopCloser(jsonR))
 	if err != nil {
 		return fmt.Errorf("building request for %s: %v", url, err)
 	}
diff --git a/services/keep-web/handler.go b/services/keep-web/handler.go
index 563a59df01..643ca4f587 100644
--- a/services/keep-web/handler.go
+++ b/services/keep-web/handler.go
@@ -76,7 +76,9 @@ func parseCollectionIDFromURL(s string) string {
 }
 
 func (h *handler) setup() {
-	h.clientPool = arvadosclient.MakeClientPool()
+	// Errors will be handled at the client pool.
+	arv, _ := arvados.NewClientFromConfig(h.Config.cluster)
+	h.clientPool = arvadosclient.MakeClientPoolWith(arv)
 
 	keepclient.RefreshServiceDiscoveryOnSIGHUP()
 	keepclient.DefaultBlockCache.MaxBlocks = h.Config.cluster.Collections.WebDAVCache.MaxBlockEntries
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 2b15d79940..0191e5ba45 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -116,6 +116,12 @@ func run(logger log.FieldLogger, cluster *arvados.Cluster) error {
 		return fmt.Errorf("Error setting up arvados client %v", err)
 	}
 
+	// If a config file is available, use the keepstores defined there
+	// instead of the legacy autodiscover mechanism via the API server
+	for k := range cluster.Services.Keepstore.InternalURLs {
+		arv.KeepServiceURIs = append(arv.KeepServiceURIs, strings.TrimRight(k.String(), "/"))
+	}
+
 	if cluster.SystemLogs.LogLevel == "debug" {
 		keepclient.DebugPrintf = log.Printf
 	}
@@ -157,7 +163,7 @@ func run(logger log.FieldLogger, cluster *arvados.Cluster) error {
 	signal.Notify(term, syscall.SIGINT)
 
 	// Start serving requests.
-	router = MakeRESTRouter(kc, time.Duration(cluster.API.KeepServiceRequestTimeout), cluster.ManagementToken)
+	router = MakeRESTRouter(kc, time.Duration(keepclient.DefaultProxyRequestTimeout), cluster.ManagementToken)
 	return http.Serve(listener, httpserver.AddRequestIDs(httpserver.LogRequests(router)))
 }
 
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index aa32356806..94ed05bff1 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -40,6 +40,12 @@ var _ = Suite(&ServerRequiredSuite{})
 // Tests that require the Keep server running
 type ServerRequiredSuite struct{}
 
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredConfigYmlSuite{})
+
+// Tests that require the Keep servers running as defined in config.yml
+type ServerRequiredConfigYmlSuite struct{}
+
 // Gocheck boilerplate
 var _ = Suite(&NoKeepServerSuite{})
 
@@ -83,6 +89,21 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
 	arvadostest.StopAPI()
 }
 
+func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
+	arvadostest.StartAPI()
+	// config.yml defines 4 keepstores
+	arvadostest.StartKeep(4, false)
+}
+
+func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
+	arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
+	arvadostest.StopKeep(4)
+	arvadostest.StopAPI()
+}
+
 func (s *NoKeepServerSuite) SetUpSuite(c *C) {
 	arvadostest.StartAPI()
 	// We need API to have some keep services listed, but the
@@ -99,12 +120,17 @@ func (s *NoKeepServerSuite) TearDownSuite(c *C) {
 	arvadostest.StopAPI()
 }
 
-func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
+func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
 	cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
 	c.Assert(err, Equals, nil)
 	cluster, err := cfg.GetCluster("")
 	c.Assert(err, Equals, nil)
 
+	if !loadKeepstoresFromConfig {
+		// Do not load Keepstore InternalURLs from the config file
+		cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
+	}
+
 	cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
 
 	listener = nil
@@ -131,7 +157,7 @@ func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
 }
 
 func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
-	runProxy(c, false)
+	runProxy(c, false, false)
 	defer closeListener()
 
 	req, err := http.NewRequest("POST",
@@ -158,7 +184,7 @@ func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	sr := map[string]string{
@@ -176,7 +202,7 @@ func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	// Set up fake keepstore to record request headers
@@ -203,7 +229,7 @@ func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	content := []byte("TestDesiredReplicas")
@@ -220,7 +246,7 @@ func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	content := []byte("TestPutWrongContentLength")
@@ -259,7 +285,7 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 	router.(*proxyHandler).timeout = time.Nanosecond
 
@@ -286,7 +312,7 @@ func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -363,7 +389,7 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
-	kc := runProxy(c, true)
+	kc := runProxy(c, true, false)
 	defer closeListener()
 
 	hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
@@ -389,7 +415,7 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
-	runProxy(c, false)
+	runProxy(c, false, false)
 	defer closeListener()
 
 	{
@@ -420,7 +446,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
-	runProxy(c, false)
+	runProxy(c, false, false)
 	defer closeListener()
 
 	{
@@ -463,7 +489,22 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
 //   With a valid but non-existing prefix (expect "\n")
 //   With an invalid prefix (expect error)
 func (s *ServerRequiredSuite) TestGetIndex(c *C) {
-	kc := runProxy(c, false)
+	getIndexWorker(c, false)
+}
+
+// Test GetIndex
+//   Uses config.yml
+//   Put one block, with 2 replicas
+//   With no prefix (expect the block locator, twice)
+//   With an existing prefix (expect the block locator, twice)
+//   With a valid but non-existing prefix (expect "\n")
+//   With an invalid prefix (expect error)
+func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
+	getIndexWorker(c, true)
+}
+
+func getIndexWorker(c *C, useConfig bool) {
+	kc := runProxy(c, false, useConfig)
 	defer closeListener()
 
 	// Put "index-data" blocks
@@ -526,7 +567,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 	hash, _, err := kc.PutB([]byte("shareddata"))
 	c.Check(err, IsNil)
@@ -539,7 +580,7 @@ func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	// Put a test block
@@ -576,7 +617,7 @@ func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	// Point keepproxy at a non-existent keepstore
@@ -602,7 +643,7 @@ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
 }
 
 func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -625,7 +666,7 @@ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
 }
 
 func (s *ServerRequiredSuite) TestPing(c *C) {
-	kc := runProxy(c, false)
+	kc := runProxy(c, false, false)
 	defer closeListener()
 
 	rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
diff --git a/services/keepstore/handler_test.go b/services/keepstore/handler_test.go
index f1a8c29255..17ed6402ce 100644
--- a/services/keepstore/handler_test.go
+++ b/services/keepstore/handler_test.go
@@ -318,6 +318,57 @@ func (s *HandlerSuite) TestPutAndDeleteSkipReadonlyVolumes(c *check.C) {
 	}
 }
 
+// Test TOUCH requests.
+func (s *HandlerSuite) TestTouchHandler(c *check.C) {
+	c.Assert(s.handler.setup(context.Background(), s.cluster, "", prometheus.NewRegistry(), testServiceURL), check.IsNil)
+	vols := s.handler.volmgr.AllWritable()
+	vols[0].Put(context.Background(), TestHash, TestBlock)
+	vols[0].Volume.(*MockVolume).TouchWithDate(TestHash, time.Now().Add(-time.Hour))
+	afterPut := time.Now()
+	t, err := vols[0].Mtime(TestHash)
+	c.Assert(err, check.IsNil)
+	c.Assert(t.Before(afterPut), check.Equals, true)
+
+	ExpectStatusCode(c,
+		"touch with no credentials",
+		http.StatusUnauthorized,
+		IssueRequest(s.handler, &RequestTester{
+			method: "TOUCH",
+			uri:    "/" + TestHash,
+		}))
+
+	ExpectStatusCode(c,
+		"touch with non-root credentials",
+		http.StatusUnauthorized,
+		IssueRequest(s.handler, &RequestTester{
+			method:   "TOUCH",
+			uri:      "/" + TestHash,
+			apiToken: arvadostest.ActiveTokenV2,
+		}))
+
+	ExpectStatusCode(c,
+		"touch non-existent block",
+		http.StatusNotFound,
+		IssueRequest(s.handler, &RequestTester{
+			method:   "TOUCH",
+			uri:      "/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+			apiToken: s.cluster.SystemRootToken,
+		}))
+
+	beforeTouch := time.Now()
+	ExpectStatusCode(c,
+		"touch block",
+		http.StatusOK,
+		IssueRequest(s.handler, &RequestTester{
+			method:   "TOUCH",
+			uri:      "/" + TestHash,
+			apiToken: s.cluster.SystemRootToken,
+		}))
+	t, err = vols[0].Mtime(TestHash)
+	c.Assert(err, check.IsNil)
+	c.Assert(t.After(beforeTouch), check.Equals, true)
+}
+
 // Test /index requests:
 //   - unauthenticated /index request
 //   - unauthenticated /index/prefix request
diff --git a/services/keepstore/handlers.go b/services/keepstore/handlers.go
index 3d0f893d82..eb0ea5ad2f 100644
--- a/services/keepstore/handlers.go
+++ b/services/keepstore/handlers.go
@@ -66,6 +66,8 @@ func MakeRESTRouter(ctx context.Context, cluster *arvados.Cluster, reg *promethe
 	// List blocks stored here whose hash has the given prefix.
 	// Privileged client only.
 	rtr.HandleFunc(`/index/{prefix:[0-9a-f]{0,32}}`, rtr.handleIndex).Methods("GET", "HEAD")
+	// Update timestamp on existing block. Privileged client only.
+	rtr.HandleFunc(`/{hash:[0-9a-f]{32}}`, rtr.handleTOUCH).Methods("TOUCH")
 
 	// Internals/debugging info (runtime.MemStats)
 	rtr.HandleFunc(`/debug.json`, rtr.DebugHandler).Methods("GET", "HEAD")
@@ -191,6 +193,34 @@ func getBufferWithContext(ctx context.Context, bufs *bufferPool, bufSize int) ([
 	}
 }
 
+func (rtr *router) handleTOUCH(resp http.ResponseWriter, req *http.Request) {
+	if !rtr.isSystemAuth(GetAPIToken(req)) {
+		http.Error(resp, UnauthorizedError.Error(), UnauthorizedError.HTTPCode)
+		return
+	}
+	hash := mux.Vars(req)["hash"]
+	vols := rtr.volmgr.AllWritable()
+	if len(vols) == 0 {
+		http.Error(resp, "no volumes", http.StatusNotFound)
+		return
+	}
+	var err error
+	for _, mnt := range vols {
+		err = mnt.Touch(hash)
+		if err == nil {
+			break
+		}
+	}
+	switch {
+	case err == nil:
+		return
+	case os.IsNotExist(err):
+		http.Error(resp, err.Error(), http.StatusNotFound)
+	default:
+		http.Error(resp, err.Error(), http.StatusInternalServerError)
+	}
+}
+
 func (rtr *router) handlePUT(resp http.ResponseWriter, req *http.Request) {
 	ctx, cancel := contextForResponse(context.TODO(), resp)
 	defer cancel()
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 80aa5ec3bb..96f2e7db39 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -129,20 +129,9 @@ func s3regions() (okList []string) {
 
 // S3Volume implements Volume using an S3 bucket.
 type S3Volume struct {
-	AccessKey          string
-	SecretKey          string
-	AuthToken          string    // populated automatically when IAMRole is used
-	AuthExpiration     time.Time // populated automatically when IAMRole is used
-	IAMRole            string
-	Endpoint           string
-	Region             string
-	Bucket             string
-	LocationConstraint bool
-	IndexPageSize      int
-	ConnectTimeout     arvados.Duration
-	ReadTimeout        arvados.Duration
-	RaceWindow         arvados.Duration
-	UnsafeDelete       bool
+	arvados.S3VolumeDriverParameters
+	AuthToken      string    // populated automatically when IAMRole is used
+	AuthExpiration time.Time // populated automatically when IAMRole is used
 
 	cluster   *arvados.Cluster
 	volume    arvados.Volume
@@ -188,8 +177,7 @@ func (v *S3Volume) bootstrapIAMCredentials() error {
 func (v *S3Volume) newS3Client() *s3.S3 {
 	auth := aws.NewAuth(v.AccessKey, v.SecretKey, v.AuthToken, v.AuthExpiration)
 	client := s3.New(*auth, v.region)
-	if v.region.EC2Endpoint.Signer == aws.V4Signature {
-		// Currently affects only eu-central-1
+	if !v.V2Signature {
 		client.Signature = aws.V4Signature
 	}
 	client.ConnectTimeout = time.Duration(v.ConnectTimeout)
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index 2c5cdf5b99..2736f00b74 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -101,6 +101,53 @@ func (s *StubbedS3Suite) TestIndex(c *check.C) {
 	}
 }
 
+func (s *StubbedS3Suite) TestSignatureVersion(c *check.C) {
+	var header http.Header
+	stub := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		header = r.Header
+	}))
+	defer stub.Close()
+
+	// Default V4 signature
+	vol := S3Volume{
+		S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+			AccessKey: "xxx",
+			SecretKey: "xxx",
+			Endpoint:  stub.URL,
+			Region:    "test-region-1",
+			Bucket:    "test-bucket-name",
+		},
+		cluster: s.cluster,
+		logger:  ctxlog.TestLogger(c),
+		metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
+	}
+	err := vol.check()
+	c.Check(err, check.IsNil)
+	err = vol.Put(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo"))
+	c.Check(err, check.IsNil)
+	c.Check(header.Get("Authorization"), check.Matches, `AWS4-HMAC-SHA256 .*`)
+
+	// Force V2 signature
+	vol = S3Volume{
+		S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+			AccessKey:   "xxx",
+			SecretKey:   "xxx",
+			Endpoint:    stub.URL,
+			Region:      "test-region-1",
+			Bucket:      "test-bucket-name",
+			V2Signature: true,
+		},
+		cluster: s.cluster,
+		logger:  ctxlog.TestLogger(c),
+		metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
+	}
+	err = vol.check()
+	c.Check(err, check.IsNil)
+	err = vol.Put(context.Background(), "acbd18db4cc2f85cedef654fccc4a4d8", []byte("foo"))
+	c.Check(err, check.IsNil)
+	c.Check(header.Get("Authorization"), check.Matches, `AWS xxx:.*`)
+}
+
 func (s *StubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
 	s.metadata = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		upd := time.Now().UTC().Add(-time.Hour).Format(time.RFC3339)
@@ -122,13 +169,15 @@ func (s *StubbedS3Suite) TestIAMRoleCredentials(c *check.C) {
 		w.WriteHeader(http.StatusNotFound)
 	}))
 	deadv := &S3Volume{
-		IAMRole:  s.metadata.URL + "/fake-metadata/test-role",
-		Endpoint: "http://localhost:12345",
-		Region:   "test-region-1",
-		Bucket:   "test-bucket-name",
-		cluster:  s.cluster,
-		logger:   ctxlog.TestLogger(c),
-		metrics:  newVolumeMetricsVecs(prometheus.NewRegistry()),
+		S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+			IAMRole:  s.metadata.URL + "/fake-metadata/test-role",
+			Endpoint: "http://localhost:12345",
+			Region:   "test-region-1",
+			Bucket:   "test-bucket-name",
+		},
+		cluster: s.cluster,
+		logger:  ctxlog.TestLogger(c),
+		metrics: newVolumeMetricsVecs(prometheus.NewRegistry()),
 	}
 	err := deadv.check()
 	c.Check(err, check.ErrorMatches, `.*/fake-metadata/test-role.*`)
@@ -468,19 +517,21 @@ func (s *StubbedS3Suite) newTestableVolume(c *check.C, cluster *arvados.Cluster,
 
 	v := &TestableS3Volume{
 		S3Volume: &S3Volume{
-			AccessKey:          accessKey,
-			SecretKey:          secretKey,
-			IAMRole:            iamRole,
-			Bucket:             TestBucketName,
-			Endpoint:           endpoint,
-			Region:             "test-region-1",
-			LocationConstraint: true,
-			UnsafeDelete:       true,
-			IndexPageSize:      1000,
-			cluster:            cluster,
-			volume:             volume,
-			logger:             ctxlog.TestLogger(c),
-			metrics:            metrics,
+			S3VolumeDriverParameters: arvados.S3VolumeDriverParameters{
+				IAMRole:            iamRole,
+				AccessKey:          accessKey,
+				SecretKey:          secretKey,
+				Bucket:             TestBucketName,
+				Endpoint:           endpoint,
+				Region:             "test-region-1",
+				LocationConstraint: true,
+				UnsafeDelete:       true,
+				IndexPageSize:      1000,
+			},
+			cluster: cluster,
+			volume:  volume,
+			logger:  ctxlog.TestLogger(c),
+			metrics: metrics,
 		},
 		c:           c,
 		server:      srv,
diff --git a/services/keepstore/unix_volume.go b/services/keepstore/unix_volume.go
index ceccd11c92..1706473cc8 100644
--- a/services/keepstore/unix_volume.go
+++ b/services/keepstore/unix_volume.go
@@ -172,10 +172,10 @@ func (v *UnixVolume) Touch(loc string) error {
 		return e
 	}
 	defer v.unlockfile(f)
-	ts := syscall.NsecToTimespec(time.Now().UnixNano())
+	ts := time.Now()
 	v.os.stats.TickOps("utimes")
 	v.os.stats.Tick(&v.os.stats.UtimesOps)
-	err = syscall.UtimesNano(p, []syscall.Timespec{ts, ts})
+	err = os.Chtimes(p, ts, ts)
 	v.os.stats.TickErr(err)
 	return err
 }
@@ -298,6 +298,19 @@ func (v *UnixVolume) WriteBlock(ctx context.Context, loc string, rdr io.Reader)
 		v.os.Remove(tmpfile.Name())
 		return err
 	}
+	// ext4 uses a low-precision clock and effectively backdates
+	// files by up to 10 ms, sometimes across a 1-second boundary,
+	// which produces confusing results in logs and tests.  We
+	// avoid this by setting the output file's timestamps
+	// explicitly, using a higher resolution clock.
+	ts := time.Now()
+	v.os.stats.TickOps("utimes")
+	v.os.stats.Tick(&v.os.stats.UtimesOps)
+	if err = os.Chtimes(tmpfile.Name(), ts, ts); err != nil {
+		err = fmt.Errorf("error setting timestamps on %s: %s", tmpfile.Name(), err)
+		v.os.Remove(tmpfile.Name())
+		return err
+	}
 	if err := v.os.Rename(tmpfile.Name(), bpath); err != nil {
 		err = fmt.Errorf("error renaming %s to %s: %s", tmpfile.Name(), bpath, err)
 		v.os.Remove(tmpfile.Name())
@@ -686,10 +699,20 @@ func (v *UnixVolume) EmptyTrash() {
 	err := filepath.Walk(v.Root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			v.logger.WithError(err).Errorf("EmptyTrash: filepath.Walk(%q) failed", path)
+			// Don't give up -- keep walking other
+			// files/dirs
+			return nil
+		} else if !info.Mode().IsDir() {
+			todo <- dirent{path, info}
 			return nil
+		} else if path == v.Root || blockDirRe.MatchString(info.Name()) {
+			// Descend into a directory that we might have
+			// put trash in.
+			return nil
+		} else {
+			// Don't descend into other dirs.
+			return filepath.SkipDir
 		}
-		todo <- dirent{path, info}
-		return nil
 	})
 	close(todo)
 	wg.Wait()
diff --git a/services/keepstore/unix_volume_test.go b/services/keepstore/unix_volume_test.go
index 7777363b9d..6b42dbc519 100644
--- a/services/keepstore/unix_volume_test.go
+++ b/services/keepstore/unix_volume_test.go
@@ -405,13 +405,13 @@ func (s *UnixVolumeSuite) TestStats(c *check.C) {
 	c.Check(stats(), check.Matches, `.*"OutBytes":3,.*`)
 	c.Check(stats(), check.Matches, `.*"CreateOps":1,.*`)
 	c.Check(stats(), check.Matches, `.*"OpenOps":0,.*`)
-	c.Check(stats(), check.Matches, `.*"UtimesOps":0,.*`)
+	c.Check(stats(), check.Matches, `.*"UtimesOps":1,.*`)
 
 	err = vol.Touch(loc)
 	c.Check(err, check.IsNil)
 	c.Check(stats(), check.Matches, `.*"FlockOps":1,.*`)
 	c.Check(stats(), check.Matches, `.*"OpenOps":1,.*`)
-	c.Check(stats(), check.Matches, `.*"UtimesOps":1,.*`)
+	c.Check(stats(), check.Matches, `.*"UtimesOps":2,.*`)
 
 	_, err = vol.Get(context.Background(), loc, make([]byte, 3))
 	c.Check(err, check.IsNil)
@@ -424,3 +424,26 @@ func (s *UnixVolumeSuite) TestStats(c *check.C) {
 	c.Check(err, check.IsNil)
 	c.Check(stats(), check.Matches, `.*"FlockOps":2,.*`)
 }
+
+func (s *UnixVolumeSuite) TestSkipUnusedDirs(c *check.C) {
+	vol := s.newTestableUnixVolume(c, s.cluster, arvados.Volume{Replication: 1}, s.metrics, false)
+
+	err := os.Mkdir(vol.UnixVolume.Root+"/aaa", 0777)
+	c.Assert(err, check.IsNil)
+	err = os.Mkdir(vol.UnixVolume.Root+"/.aaa", 0777) // EmptyTrash should not look here
+	c.Assert(err, check.IsNil)
+	deleteme := vol.UnixVolume.Root + "/aaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.trash.1"
+	err = ioutil.WriteFile(deleteme, []byte{1, 2, 3}, 0777)
+	c.Assert(err, check.IsNil)
+	skipme := vol.UnixVolume.Root + "/.aaa/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa.trash.1"
+	err = ioutil.WriteFile(skipme, []byte{1, 2, 3}, 0777)
+	c.Assert(err, check.IsNil)
+	vol.EmptyTrash()
+
+	_, err = os.Stat(skipme)
+	c.Check(err, check.IsNil)
+
+	_, err = os.Stat(deleteme)
+	c.Check(err, check.NotNil)
+	c.Check(os.IsNotExist(err), check.Equals, true)
+}
diff --git a/services/keepstore/volume_test.go b/services/keepstore/volume_test.go
index a928a71a2c..2de21edde6 100644
--- a/services/keepstore/volume_test.go
+++ b/services/keepstore/volume_test.go
@@ -178,13 +178,20 @@ func (v *MockVolume) Put(ctx context.Context, loc string, block []byte) error {
 }
 
 func (v *MockVolume) Touch(loc string) error {
+	return v.TouchWithDate(loc, time.Now())
+}
+
+func (v *MockVolume) TouchWithDate(loc string, t time.Time) error {
 	v.gotCall("Touch")
 	<-v.Gate
 	if v.volume.ReadOnly {
 		return MethodDisabledError
 	}
+	if _, exists := v.Store[loc]; !exists {
+		return os.ErrNotExist
+	}
 	if v.Touchable {
-		v.Timestamps[loc] = time.Now()
+		v.Timestamps[loc] = t
 		return nil
 	}
 	return errors.New("Touch failed")
diff --git a/services/ws/service_test.go b/services/ws/service_test.go
index 7213dcad2a..13726836a4 100644
--- a/services/ws/service_test.go
+++ b/services/ws/service_test.go
@@ -198,7 +198,7 @@ ManagementToken: qqqqq
 	c.Check(err, check.IsNil)
 	c.Check(cluster, check.NotNil)
 
-	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com"})
+	c.Check(cluster.Services.Controller.ExternalURL, check.Equals, arvados.URL{Scheme: "https", Host: "example.com", Path: "/"})
 	c.Check(cluster.SystemRootToken, check.Equals, "abcdefg")
 
 	c.Check(cluster.PostgreSQL.Connection, check.DeepEquals, arvados.PostgreSQLConnection{
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index af9824c3a8..59aca1e5b4 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -236,7 +236,7 @@ run() {
         mkdir -p "$PG_DATA" "$VAR_DATA" "$PASSENGER" "$GEMS" "$PIPCACHE" "$NPMCACHE" "$GOSTUFF" "$RLIBS"
 
         if ! test -d "$ARVADOS_ROOT" ; then
-            git clone https://github.com/arvados/arvados.git "$ARVADOS_ROOT"
+            git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
         fi
         if ! test -d "$SSO_ROOT" ; then
             git clone https://github.com/arvados/sso-devise-omniauth-provider.git "$SSO_ROOT"
@@ -614,6 +614,7 @@ sv stop keepstore0
 sv stop keepstore1
 sv stop keepproxy
 cd /usr/src/arvados/services/api
+export DISABLE_DATABASE_ENVIRONMENT_CHECK=1
 export RAILS_ENV=development
 bundle exec rake db:drop
 rm /var/lib/arvados/api_database_setup
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index c459260ace..34d3845eaf 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -43,6 +43,6 @@ RUN sudo -u arvbox /var/lib/arvbox/service/vm/run-service --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/keepproxy/run-service --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/arv-git-httpd/run-service --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/crunch-dispatch-local/run-service --only-deps
-RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run-service --only-deps
+RUN sudo -u arvbox /var/lib/arvbox/service/websockets/run --only-deps
 RUN sudo -u arvbox /usr/local/lib/arvbox/keep-setup.sh --only-deps
 RUN sudo -u arvbox /var/lib/arvbox/service/sdk/run-service
diff --git a/tools/arvbox/lib/arvbox/docker/cluster-config.sh b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
index ed4795d1cc..4798cb6ccd 100755
--- a/tools/arvbox/lib/arvbox/docker/cluster-config.sh
+++ b/tools/arvbox/lib/arvbox/docker/cluster-config.sh
@@ -139,8 +139,10 @@ Clusters:
       DefaultReplication: 1
       TrustAllContent: true
     Login:
-      ProviderAppSecret: $sso_app_secret
-      ProviderAppID: arvados-server
+      SSO:
+        Enable: true
+        ProviderAppSecret: $sso_app_secret
+        ProviderAppID: arvados-server
     Users:
       NewUsersAreActive: true
       AutoAdminFirstUser: true
diff --git a/tools/arvbox/lib/arvbox/docker/common.sh b/tools/arvbox/lib/arvbox/docker/common.sh
index 9c933e870f..89864d5d18 100644
--- a/tools/arvbox/lib/arvbox/docker/common.sh
+++ b/tools/arvbox/lib/arvbox/docker/common.sh
@@ -88,12 +88,12 @@ pip_install() {
     popd
 
     if [ "$PYCMD" = "python3" ]; then
-	if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
+        if ! pip3 install --prefix /usr/local --no-index --find-links /var/lib/pip $1 ; then
             pip3 install --prefix /usr/local $1
-	fi
+        fi
     else
-	if ! pip install --no-index --find-links /var/lib/pip $1 ; then
+        if ! pip install --no-index --find-links /var/lib/pip $1 ; then
             pip install $1
-	fi
+        fi
     fi
 }
diff --git a/tools/arvbox/lib/arvbox/docker/service/controller/run b/tools/arvbox/lib/arvbox/docker/service/controller/run
index 7c16e08e2e..588e9d2dad 100755
--- a/tools/arvbox/lib/arvbox/docker/service/controller/run
+++ b/tools/arvbox/lib/arvbox/docker/service/controller/run
@@ -17,4 +17,4 @@ fi
 
 /usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
 
-exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-controller
+exec /usr/local/bin/arvados-controller
diff --git a/tools/arvbox/lib/arvbox/docker/service/websockets/run b/tools/arvbox/lib/arvbox/docker/service/websockets/run
deleted file mode 120000
index a388c8b67b..0000000000
--- a/tools/arvbox/lib/arvbox/docker/service/websockets/run
+++ /dev/null
@@ -1 +0,0 @@
-/usr/local/lib/arvbox/runsu.sh
\ No newline at end of file
diff --git a/tools/arvbox/lib/arvbox/docker/service/websockets/run b/tools/arvbox/lib/arvbox/docker/service/websockets/run
new file mode 100755
index 0000000000..efa2e08a7a
--- /dev/null
+++ b/tools/arvbox/lib/arvbox/docker/service/websockets/run
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+exec 2>&1
+set -ex -o pipefail
+
+. /usr/local/lib/arvbox/common.sh
+. /usr/local/lib/arvbox/go-setup.sh
+
+(cd /usr/local/bin && ln -sf arvados-server arvados-ws)
+
+if test "$1" = "--only-deps" ; then
+    exit
+fi
+
+/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
+
+exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-ws
diff --git a/tools/arvbox/lib/arvbox/docker/service/websockets/run-service b/tools/arvbox/lib/arvbox/docker/service/websockets/run-service
deleted file mode 100755
index efa2e08a7a..0000000000
--- a/tools/arvbox/lib/arvbox/docker/service/websockets/run-service
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-exec 2>&1
-set -ex -o pipefail
-
-. /usr/local/lib/arvbox/common.sh
-. /usr/local/lib/arvbox/go-setup.sh
-
-(cd /usr/local/bin && ln -sf arvados-server arvados-ws)
-
-if test "$1" = "--only-deps" ; then
-    exit
-fi
-
-/usr/local/lib/arvbox/runsu.sh flock /var/lib/arvados/cluster_config.yml.lock /usr/local/lib/arvbox/cluster-config.sh
-
-exec /usr/local/lib/arvbox/runsu.sh /usr/local/bin/arvados-ws
diff --git a/tools/keep-exercise/keep-exercise.go b/tools/keep-exercise/keep-exercise.go
index 8fec51aeb9..163291c238 100644
--- a/tools/keep-exercise/keep-exercise.go
+++ b/tools/keep-exercise/keep-exercise.go
@@ -28,6 +28,8 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
 	"git.arvados.org/arvados.git/sdk/go/arvadosclient"
@@ -48,6 +50,7 @@ var (
 	ServiceURL    = flag.String("url", "", "specify scheme://host of a single keep service to exercise (instead of using all advertised services like normal clients)")
 	ServiceUUID   = flag.String("uuid", "", "specify UUID of a single advertised keep service to exercise")
 	getVersion    = flag.Bool("version", false, "Print version information and exit.")
+	RunTime       = flag.Duration("run-time", 0, "time to run (e.g. 60s), or 0 to run indefinitely (default)")
 )
 
 func main() {
@@ -59,15 +62,15 @@ func main() {
 		os.Exit(0)
 	}
 
-	log.Printf("keep-exercise %s started", version)
+	stderr := log.New(os.Stderr, "", log.LstdFlags)
 
 	arv, err := arvadosclient.MakeArvadosClient()
 	if err != nil {
-		log.Fatal(err)
+		stderr.Fatal(err)
 	}
 	kc, err := keepclient.MakeKeepClient(arv)
 	if err != nil {
-		log.Fatal(err)
+		stderr.Fatal(err)
 	}
 	kc.Want_replicas = *Replicas
 
@@ -78,18 +81,18 @@ func main() {
 		Transport: &transport,
 	}
 
-	overrideServices(kc)
+	overrideServices(kc, stderr)
 
 	nextLocator := make(chan string, *ReadThreads+*WriteThreads)
 
-	go countBeans(nextLocator)
+	go countBeans(nextLocator, stderr)
 	for i := 0; i < *WriteThreads; i++ {
 		nextBuf := make(chan []byte, 1)
-		go makeBufs(nextBuf, i)
-		go doWrites(kc, nextBuf, nextLocator)
+		go makeBufs(nextBuf, i, stderr)
+		go doWrites(kc, nextBuf, nextLocator, stderr)
 	}
 	for i := 0; i < *ReadThreads; i++ {
-		go doReads(kc, nextLocator)
+		go doReads(kc, nextLocator, stderr)
 	}
 	<-make(chan struct{})
 }
@@ -101,25 +104,37 @@ var bytesOutChan = make(chan uint64)
 // Send struct{}{} to errorsChan when an error happens.
 var errorsChan = make(chan struct{})
 
-func countBeans(nextLocator chan string) {
+func countBeans(nextLocator chan string, stderr *log.Logger) {
 	t0 := time.Now()
 	var tickChan <-chan time.Time
+	var endChan <-chan time.Time
+	c := make(chan os.Signal)
+	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
 	if *StatsInterval > 0 {
 		tickChan = time.NewTicker(*StatsInterval).C
 	}
+	if *RunTime > 0 {
+		endChan = time.NewTicker(*RunTime).C
+	}
 	var bytesIn uint64
 	var bytesOut uint64
 	var errors uint64
+	var rateIn, rateOut float64
+	var maxRateIn, maxRateOut float64
+	var abort, printCsv bool
+	csv := log.New(os.Stdout, "", 0)
+	csv.Println("Timestamp,Elapsed,Read (bytes),Avg Read Speed (MiB/s),Peak Read Speed (MiB/s),Written (bytes),Avg Write Speed (MiB/s),Peak Write Speed (MiB/s),Errors,ReadThreads,WriteThreads,VaryRequest,VaryThread,BlockSize,Replicas,StatsInterval,ServiceURL,ServiceUUID,RunTime")
 	for {
 		select {
 		case <-tickChan:
-			elapsed := time.Since(t0)
-			log.Printf("%v elapsed: read %v bytes (%.1f MiB/s), wrote %v bytes (%.1f MiB/s), errors %d",
-				elapsed,
-				bytesIn, (float64(bytesIn) / elapsed.Seconds() / 1048576),
-				bytesOut, (float64(bytesOut) / elapsed.Seconds() / 1048576),
-				errors,
-			)
+			printCsv = true
+		case <-endChan:
+			printCsv = true
+			abort = true
+		case <-c:
+			printCsv = true
+			abort = true
+			fmt.Print("\r") // Suppress the ^C print
 		case i := <-bytesInChan:
 			bytesIn += i
 		case o := <-bytesOutChan:
@@ -127,10 +142,42 @@ func countBeans(nextLocator chan string) {
 		case <-errorsChan:
 			errors++
 		}
+		if printCsv {
+			elapsed := time.Since(t0)
+			rateIn = float64(bytesIn) / elapsed.Seconds() / 1048576
+			if rateIn > maxRateIn {
+				maxRateIn = rateIn
+			}
+			rateOut = float64(bytesOut) / elapsed.Seconds() / 1048576
+			if rateOut > maxRateOut {
+				maxRateOut = rateOut
+			}
+			csv.Printf("%v,%v,%v,%.1f,%.1f,%v,%.1f,%.1f,%d,%d,%d,%t,%t,%d,%d,%s,%s,%s,%s",
+				time.Now().Format("2006-01-02 15:04:05"),
+				elapsed,
+				bytesIn, rateIn, maxRateIn,
+				bytesOut, rateOut, maxRateOut,
+				errors,
+				*ReadThreads,
+				*WriteThreads,
+				*VaryRequest,
+				*VaryThread,
+				*BlockSize,
+				*Replicas,
+				*StatsInterval,
+				*ServiceURL,
+				*ServiceUUID,
+				*RunTime,
+			)
+			printCsv = false
+		}
+		if abort {
+			os.Exit(0)
+		}
 	}
 }
 
-func makeBufs(nextBuf chan<- []byte, threadID int) {
+func makeBufs(nextBuf chan<- []byte, threadID int, stderr *log.Logger) {
 	buf := make([]byte, *BlockSize)
 	if *VaryThread {
 		binary.PutVarint(buf, int64(threadID))
@@ -143,7 +190,7 @@ func makeBufs(nextBuf chan<- []byte, threadID int) {
 		if *VaryRequest {
 			rnd := make([]byte, randSize)
 			if _, err := io.ReadFull(rand.Reader, rnd); err != nil {
-				log.Fatal(err)
+				stderr.Fatal(err)
 			}
 			buf = append(rnd, buf[randSize:]...)
 		}
@@ -151,11 +198,11 @@ func makeBufs(nextBuf chan<- []byte, threadID int) {
 	}
 }
 
-func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string) {
+func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan<- string, stderr *log.Logger) {
 	for buf := range nextBuf {
 		locator, _, err := kc.PutB(buf)
 		if err != nil {
-			log.Print(err)
+			stderr.Print(err)
 			errorsChan <- struct{}{}
 			continue
 		}
@@ -168,18 +215,18 @@ func doWrites(kc *keepclient.KeepClient, nextBuf <-chan []byte, nextLocator chan
 	}
 }
 
-func doReads(kc *keepclient.KeepClient, nextLocator <-chan string) {
+func doReads(kc *keepclient.KeepClient, nextLocator <-chan string, stderr *log.Logger) {
 	for locator := range nextLocator {
 		rdr, size, url, err := kc.Get(locator)
 		if err != nil {
-			log.Print(err)
+			stderr.Print(err)
 			errorsChan <- struct{}{}
 			continue
 		}
 		n, err := io.Copy(ioutil.Discard, rdr)
 		rdr.Close()
 		if n != size || err != nil {
-			log.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
+			stderr.Printf("Got %d bytes (expected %d) from %s: %v", n, size, url, err)
 			errorsChan <- struct{}{}
 			continue
 			// Note we don't count the bytes received in
@@ -190,7 +237,7 @@ func doReads(kc *keepclient.KeepClient, nextLocator <-chan string) {
 	}
 }
 
-func overrideServices(kc *keepclient.KeepClient) {
+func overrideServices(kc *keepclient.KeepClient, stderr *log.Logger) {
 	roots := make(map[string]string)
 	if *ServiceURL != "" {
 		roots["zzzzz-bi6l4-000000000000000"] = *ServiceURL
@@ -202,7 +249,7 @@ func overrideServices(kc *keepclient.KeepClient) {
 			}
 		}
 		if len(roots) == 0 {
-			log.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
+			stderr.Fatalf("Service %q was not in list advertised by API %+q", *ServiceUUID, kc.GatewayRoots())
 		}
 	} else {
 		return
diff --git a/tools/sync-groups/sync-groups.go b/tools/sync-groups/sync-groups.go
index 7c5cd0558b..4d03ba89e3 100644
--- a/tools/sync-groups/sync-groups.go
+++ b/tools/sync-groups/sync-groups.go
@@ -26,11 +26,14 @@ type resourceList interface {
 	GetItems() []interface{}
 }
 
-// GroupInfo tracks previous and current members of a particular Group
+// GroupPermissions maps permission levels on groups (can_read, can_write, can_manage)
+type GroupPermissions map[string]bool
+
+// GroupInfo tracks previous and current member's permissions on a particular Group
 type GroupInfo struct {
 	Group           arvados.Group
-	PreviousMembers map[string]bool
-	CurrentMembers  map[string]bool
+	PreviousMembers map[string]GroupPermissions
+	CurrentMembers  map[string]GroupPermissions
 }
 
 // GetUserID returns the correct user id value depending on the selector
@@ -134,9 +137,10 @@ func ParseFlags(config *ConfigParams) error {
 
 	// Set up usage message
 	flags.Usage = func() {
-		usageStr := `Synchronize remote groups into Arvados from a CSV format file with 2 columns:
-  * 1st column: Group name
-  * 2nd column: User identifier`
+		usageStr := `Synchronize remote groups into Arvados from a CSV format file with 3 columns:
+  * 1st: Group name
+  * 2nd: User identifier
+  * 3rd (Optional): User permission on the group: can_read, can_write or can_manage. (Default: can_write)`
 		fmt.Fprintf(os.Stderr, "%s\n\n", usageStr)
 		fmt.Fprintf(os.Stderr, "Usage:\n%s [OPTIONS] <input-file.csv>\n\n", os.Args[0])
 		fmt.Fprintf(os.Stderr, "Options:\n")
@@ -222,8 +226,9 @@ func SetParentGroup(cfg *ConfigParams) error {
 				log.Println("Default parent group not found, creating...")
 			}
 			groupData := map[string]string{
-				"name":       cfg.ParentGroupName,
-				"owner_uuid": cfg.SysUserUUID,
+				"name":        cfg.ParentGroupName,
+				"owner_uuid":  cfg.SysUserUUID,
+				"group_class": "role",
 			}
 			if err := CreateGroup(cfg, &parentGroup, groupData); err != nil {
 				return fmt.Errorf("error creating system user owned group named %q: %s", groupData["name"], err)
@@ -334,16 +339,30 @@ func doMain(cfg *ConfigParams) error {
 	// Remove previous members not listed on this run
 	for groupUUID := range remoteGroups {
 		gi := remoteGroups[groupUUID]
-		evictedMembers := subtract(gi.PreviousMembers, gi.CurrentMembers)
+		evictedMemberPerms := subtract(gi.PreviousMembers, gi.CurrentMembers)
 		groupName := gi.Group.Name
-		if len(evictedMembers) > 0 {
-			log.Printf("Removing %d users from group %q", len(evictedMembers), groupName)
-		}
-		for evictedUser := range evictedMembers {
-			if err := RemoveMemberFromGroup(cfg, allUsers[userIDToUUID[evictedUser]], gi.Group); err != nil {
+		if len(evictedMemberPerms) > 0 {
+			log.Printf("Removing permissions from %d users on group %q", len(evictedMemberPerms), groupName)
+		}
+		for member := range evictedMemberPerms {
+			var perms []string
+			completeMembershipRemoval := false
+			if _, ok := gi.CurrentMembers[member]; !ok {
+				completeMembershipRemoval = true
+				membershipsRemoved++
+			} else {
+				// Collect which user->group permission links should be removed
+				for p := range evictedMemberPerms[member] {
+					if evictedMemberPerms[member][p] {
+						perms = append(perms, p)
+					}
+				}
+				membershipsRemoved += len(perms)
+			}
+			if err := RemoveMemberLinksFromGroup(cfg, allUsers[userIDToUUID[member]],
+				perms, completeMembershipRemoval, gi.Group); err != nil {
 				return err
 			}
-			membershipsRemoved++
 		}
 	}
 	log.Printf("Groups created: %d. Memberships added: %d, removed: %d, skipped: %d", groupsCreated, membershipsAdded, membershipsRemoved, membershipsSkipped)
@@ -362,7 +381,8 @@ func ProcessFile(
 ) (groupsCreated, membersAdded, membersSkipped int, err error) {
 	lineNo := 0
 	csvReader := csv.NewReader(f)
-	csvReader.FieldsPerRecord = 2
+	// Allow variable number of fields.
+	csvReader.FieldsPerRecord = -1
 	for {
 		record, e := csvReader.Read()
 		if e == io.EOF {
@@ -373,10 +393,24 @@ func ProcessFile(
 			err = fmt.Errorf("error parsing %q, line %d", cfg.Path, lineNo)
 			return
 		}
+		// Only allow 2 or 3 fields per record for backwards compatibility.
+		if len(record) < 2 || len(record) > 3 {
+			err = fmt.Errorf("error parsing %q, line %d: found %d fields but only 2 or 3 are allowed", cfg.Path, lineNo, len(record))
+			return
+		}
 		groupName := strings.TrimSpace(record[0])
 		groupMember := strings.TrimSpace(record[1]) // User ID (username or email)
-		if groupName == "" || groupMember == "" {
-			log.Printf("Warning: CSV record has at least one empty field (%s, %s). Skipping", groupName, groupMember)
+		groupPermission := "can_write"
+		if len(record) == 3 {
+			groupPermission = strings.ToLower(record[2])
+		}
+		if groupName == "" || groupMember == "" || groupPermission == "" {
+			log.Printf("Warning: CSV record has at least one empty field (%s, %s, %s). Skipping", groupName, groupMember, groupPermission)
+			membersSkipped++
+			continue
+		}
+		if !(groupPermission == "can_read" || groupPermission == "can_write" || groupPermission == "can_manage") {
+			log.Printf("Warning: 3rd field should be 'can_read', 'can_write' or 'can_manage'. Found: %q at line %d, skipping.", groupPermission, lineNo)
 			membersSkipped++
 			continue
 		}
@@ -405,26 +439,36 @@ func ProcessFile(
 			groupNameToUUID[groupName] = newGroup.UUID
 			remoteGroups[newGroup.UUID] = &GroupInfo{
 				Group:           newGroup,
-				PreviousMembers: make(map[string]bool), // Empty set
-				CurrentMembers:  make(map[string]bool), // Empty set
+				PreviousMembers: make(map[string]GroupPermissions),
+				CurrentMembers:  make(map[string]GroupPermissions),
 			}
 			groupsCreated++
 		}
 		// Both group & user exist, check if user is a member
 		groupUUID := groupNameToUUID[groupName]
 		gi := remoteGroups[groupUUID]
-		if !gi.PreviousMembers[groupMember] && !gi.CurrentMembers[groupMember] {
+		if !gi.PreviousMembers[groupMember][groupPermission] && !gi.CurrentMembers[groupMember][groupPermission] {
 			if cfg.Verbose {
 				log.Printf("Adding %q to group %q", groupMember, groupName)
 			}
-			// User wasn't a member, but should be.
-			if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group); e != nil {
+			// User permissionwasn't there, but should be. Avoid duplicating the
+			// group->user link when necessary.
+			createG2ULink := true
+			if _, ok := gi.PreviousMembers[groupMember]; ok {
+				createG2ULink = false // User is already member of the group
+			}
+			if e := AddMemberToGroup(cfg, allUsers[userIDToUUID[groupMember]], gi.Group, groupPermission, createG2ULink); e != nil {
 				err = e
 				return
 			}
 			membersAdded++
 		}
-		gi.CurrentMembers[groupMember] = true
+		if _, ok := gi.CurrentMembers[groupMember]; ok {
+			gi.CurrentMembers[groupMember][groupPermission] = true
+		} else {
+			gi.CurrentMembers[groupMember] = GroupPermissions{groupPermission: true}
+		}
+
 	}
 	return
 }
@@ -452,11 +496,17 @@ func GetAll(c *arvados.Client, res string, params arvados.ResourceListParams, pa
 	return allItems, nil
 }
 
-func subtract(setA map[string]bool, setB map[string]bool) map[string]bool {
-	result := make(map[string]bool)
+func subtract(setA map[string]GroupPermissions, setB map[string]GroupPermissions) map[string]GroupPermissions {
+	result := make(map[string]GroupPermissions)
 	for element := range setA {
-		if !setB[element] {
-			result[element] = true
+		if _, ok := setB[element]; !ok {
+			result[element] = setA[element]
+		} else {
+			for perm := range setA[element] {
+				if _, ok := setB[element][perm]; !ok {
+					result[element] = GroupPermissions{perm: true}
+				}
+			}
 		}
 	}
 	return result
@@ -479,17 +529,21 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
 
 	params := arvados.ResourceListParams{
 		Filters: []arvados.Filter{{
-			Attr:     "owner_uuid",
+			Attr:     "tail_uuid",
 			Operator: "=",
 			Operand:  cfg.ParentGroupUUID,
 		}},
 	}
-	results, err := GetAll(cfg.Client, "groups", params, &GroupList{})
+	results, err := GetAll(cfg.Client, "links", params, &LinkList{})
 	if err != nil {
 		return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote groups: %s", err)
 	}
 	for _, item := range results {
-		group := item.(arvados.Group)
+		var group arvados.Group
+		err = GetGroup(cfg, &group, item.(arvados.Link).HeadUUID)
+		if err != nil {
+			return remoteGroups, groupNameToUUID, fmt.Errorf("error getting remote group: %s", err)
+		}
 		// Group -> User filter
 		g2uFilter := arvados.ResourceListParams{
 			Filters: []arvados.Filter{{
@@ -526,8 +580,8 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
 				Operand:  "permission",
 			}, {
 				Attr:     "name",
-				Operator: "=",
-				Operand:  "can_write",
+				Operator: "in",
+				Operand:  []string{"can_read", "can_write", "can_manage"},
 			}, {
 				Attr:     "head_uuid",
 				Operator: "=",
@@ -540,18 +594,23 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
 		}
 		g2uLinks, err := GetAll(cfg.Client, "links", g2uFilter, &LinkList{})
 		if err != nil {
-			return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_read) links for group %q: %s", group.Name, err)
+			return remoteGroups, groupNameToUUID, fmt.Errorf("error getting group->user 'can_read' links for group %q: %s", group.Name, err)
 		}
 		u2gLinks, err := GetAll(cfg.Client, "links", u2gFilter, &LinkList{})
 		if err != nil {
-			return remoteGroups, groupNameToUUID, fmt.Errorf("error getting member (can_write) links for group %q: %s", group.Name, err)
+			return remoteGroups, groupNameToUUID, fmt.Errorf("error getting user->group links for group %q: %s", group.Name, err)
 		}
-		// Build a list of user ids (email or username) belonging to this group
-		membersSet := make(map[string]bool)
-		u2gLinkSet := make(map[string]bool)
+		// Build a list of user ids (email or username) belonging to this group.
+		membersSet := make(map[string]GroupPermissions)
+		u2gLinkSet := make(map[string]GroupPermissions)
 		for _, l := range u2gLinks {
-			linkedMemberUUID := l.(arvados.Link).TailUUID
-			u2gLinkSet[linkedMemberUUID] = true
+			link := l.(arvados.Link)
+			// Also save the member's group access level.
+			if _, ok := u2gLinkSet[link.TailUUID]; ok {
+				u2gLinkSet[link.TailUUID][link.Name] = true
+			} else {
+				u2gLinkSet[link.TailUUID] = GroupPermissions{link.Name: true}
+			}
 		}
 		for _, item := range g2uLinks {
 			link := item.(arvados.Link)
@@ -569,55 +628,81 @@ func GetRemoteGroups(cfg *ConfigParams, allUsers map[string]arvados.User) (remot
 			if err != nil {
 				return remoteGroups, groupNameToUUID, err
 			}
-			membersSet[memberID] = true
+			membersSet[memberID] = u2gLinkSet[link.HeadUUID]
 		}
 		remoteGroups[group.UUID] = &GroupInfo{
 			Group:           group,
 			PreviousMembers: membersSet,
-			CurrentMembers:  make(map[string]bool), // Empty set
+			CurrentMembers:  make(map[string]GroupPermissions),
 		}
 		groupNameToUUID[group.Name] = group.UUID
 	}
 	return remoteGroups, groupNameToUUID, nil
 }
 
-// RemoveMemberFromGroup remove all links related to the membership
-func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+// RemoveMemberLinksFromGroup remove all links related to the membership
+func RemoveMemberLinksFromGroup(cfg *ConfigParams, user arvados.User, linkNames []string, completeRemoval bool, group arvados.Group) error {
 	if cfg.Verbose {
 		log.Printf("Getting group membership links for user %q (%s) on group %q (%s)", user.Username, user.UUID, group.Name, group.UUID)
 	}
 	var links []interface{}
-	// Search for all group<->user links (both ways)
-	for _, filterset := range [][]arvados.Filter{
-		// Group -> User
-		{{
-			Attr:     "link_class",
-			Operator: "=",
-			Operand:  "permission",
-		}, {
-			Attr:     "tail_uuid",
-			Operator: "=",
-			Operand:  group.UUID,
-		}, {
-			Attr:     "head_uuid",
-			Operator: "=",
-			Operand:  user.UUID,
-		}},
-		// Group <- User
-		{{
-			Attr:     "link_class",
-			Operator: "=",
-			Operand:  "permission",
-		}, {
-			Attr:     "tail_uuid",
-			Operator: "=",
-			Operand:  user.UUID,
-		}, {
-			Attr:     "head_uuid",
-			Operator: "=",
-			Operand:  group.UUID,
-		}},
-	} {
+	var filters [][]arvados.Filter
+	if completeRemoval {
+		// Search for all group<->user links (both ways)
+		filters = [][]arvados.Filter{
+			// Group -> User
+			{{
+				Attr:     "link_class",
+				Operator: "=",
+				Operand:  "permission",
+			}, {
+				Attr:     "tail_uuid",
+				Operator: "=",
+				Operand:  group.UUID,
+			}, {
+				Attr:     "head_uuid",
+				Operator: "=",
+				Operand:  user.UUID,
+			}},
+			// Group <- User
+			{{
+				Attr:     "link_class",
+				Operator: "=",
+				Operand:  "permission",
+			}, {
+				Attr:     "tail_uuid",
+				Operator: "=",
+				Operand:  user.UUID,
+			}, {
+				Attr:     "head_uuid",
+				Operator: "=",
+				Operand:  group.UUID,
+			}},
+		}
+	} else {
+		// Search only for the requested Group <- User permission links
+		filters = [][]arvados.Filter{
+			{{
+				Attr:     "link_class",
+				Operator: "=",
+				Operand:  "permission",
+			}, {
+				Attr:     "tail_uuid",
+				Operator: "=",
+				Operand:  user.UUID,
+			}, {
+				Attr:     "head_uuid",
+				Operator: "=",
+				Operand:  group.UUID,
+			}, {
+				Attr:     "name",
+				Operator: "in",
+				Operand:  linkNames,
+			}},
+		}
+	}
+
+	for _, filterset := range filters {
 		l, err := GetAll(cfg.Client, "links", arvados.ResourceListParams{Filters: filterset}, &LinkList{})
 		if err != nil {
 			userID, _ := GetUserID(user, cfg.UserID)
@@ -641,29 +726,32 @@ func RemoveMemberFromGroup(cfg *ConfigParams, user arvados.User, group arvados.G
 }
 
 // AddMemberToGroup create membership links
-func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group) error {
+func AddMemberToGroup(cfg *ConfigParams, user arvados.User, group arvados.Group, perm string, createG2ULink bool) error {
 	var newLink arvados.Link
-	linkData := map[string]string{
-		"owner_uuid": cfg.SysUserUUID,
-		"link_class": "permission",
-		"name":       "can_read",
-		"tail_uuid":  group.UUID,
-		"head_uuid":  user.UUID,
-	}
-	if err := CreateLink(cfg, &newLink, linkData); err != nil {
-		userID, _ := GetUserID(user, cfg.UserID)
-		return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+	var linkData map[string]string
+	if createG2ULink {
+		linkData = map[string]string{
+			"owner_uuid": cfg.SysUserUUID,
+			"link_class": "permission",
+			"name":       "can_read",
+			"tail_uuid":  group.UUID,
+			"head_uuid":  user.UUID,
+		}
+		if err := CreateLink(cfg, &newLink, linkData); err != nil {
+			userID, _ := GetUserID(user, cfg.UserID)
+			return fmt.Errorf("error adding group %q -> user %q read permission: %s", group.Name, userID, err)
+		}
 	}
 	linkData = map[string]string{
 		"owner_uuid": cfg.SysUserUUID,
 		"link_class": "permission",
-		"name":       "can_write",
+		"name":       perm,
 		"tail_uuid":  user.UUID,
 		"head_uuid":  group.UUID,
 	}
 	if err := CreateLink(cfg, &newLink, linkData); err != nil {
 		userID, _ := GetUserID(user, cfg.UserID)
-		return fmt.Errorf("error adding user %q -> group %q write permission: %s", userID, group.Name, err)
+		return fmt.Errorf("error adding user %q -> group %q %s permission: %s", userID, group.Name, perm, err)
 	}
 	return nil
 }
diff --git a/tools/sync-groups/sync-groups_test.go b/tools/sync-groups/sync-groups_test.go
index 3ef3600797..2da8c1cdde 100644
--- a/tools/sync-groups/sync-groups_test.go
+++ b/tools/sync-groups/sync-groups_test.go
@@ -106,7 +106,7 @@ func MakeTempCSVFile(data [][]string) (f *os.File, err error) {
 }
 
 // GroupMembershipExists checks that both needed links exist between user and group
-func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string) bool {
+func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string, perm string) bool {
 	ll := LinkList{}
 	// Check Group -> User can_read permission
 	params := arvados.ResourceListParams{
@@ -145,7 +145,7 @@ func GroupMembershipExists(ac *arvados.Client, userUUID string, groupUUID string
 		}, {
 			Attr:     "name",
 			Operator: "=",
-			Operand:  "can_write",
+			Operand:  perm,
 		}, {
 			Attr:     "tail_uuid",
 			Operator: "=",
@@ -170,7 +170,7 @@ func RemoteGroupExists(cfg *ConfigParams, groupName string) (uuid string, err er
 		}, {
 			Attr:     "owner_uuid",
 			Operator: "=",
-			Operand:  cfg.ParentGroupUUID,
+			Operand:  cfg.SysUserUUID,
 		}, {
 			Attr:     "group_class",
 			Operator: "=",
@@ -259,10 +259,103 @@ func (s *TestSuite) TestIgnoreSpaces(c *C) {
 		groupUUID, err := RemoteGroupExists(s.cfg, groupName)
 		c.Assert(err, IsNil)
 		c.Assert(groupUUID, Not(Equals), "")
-		c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+		c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
 	}
 }
 
+// Error out when records have <2 or >3 records
+func (s *TestSuite) TestWrongNumberOfFields(c *C) {
+	for _, testCase := range [][][]string{
+		{{"field1"}},
+		{{"field1", "field2", "field3", "field4"}},
+		{{"field1", "field2", "field3", "field4", "field5"}},
+	} {
+		tmpfile, err := MakeTempCSVFile(testCase)
+		c.Assert(err, IsNil)
+		defer os.Remove(tmpfile.Name())
+		s.cfg.Path = tmpfile.Name()
+		err = doMain(s.cfg)
+		c.Assert(err, Not(IsNil))
+	}
+}
+
+// Check different membership permissions
+func (s *TestSuite) TestMembershipLevels(c *C) {
+	userEmail := s.users[arvadostest.ActiveUserUUID].Email
+	userUUID := s.users[arvadostest.ActiveUserUUID].UUID
+	data := [][]string{
+		{"TestGroup1", userEmail, "can_read"},
+		{"TestGroup2", userEmail, "can_write"},
+		{"TestGroup3", userEmail, "can_manage"},
+		{"TestGroup4", userEmail, "invalid_permission"},
+	}
+	tmpfile, err := MakeTempCSVFile(data)
+	c.Assert(err, IsNil)
+	defer os.Remove(tmpfile.Name()) // clean up
+	s.cfg.Path = tmpfile.Name()
+	err = doMain(s.cfg)
+	c.Assert(err, IsNil)
+	for _, record := range data {
+		groupName := record[0]
+		permLevel := record[2]
+		if permLevel != "invalid_permission" {
+			groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+			c.Assert(err, IsNil)
+			c.Assert(groupUUID, Not(Equals), "")
+			c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, permLevel), Equals, true)
+		} else {
+			groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+			c.Assert(err, IsNil)
+			c.Assert(groupUUID, Equals, "")
+		}
+	}
+}
+
+// Check membership level change
+func (s *TestSuite) TestMembershipLevelUpdate(c *C) {
+	userEmail := s.users[arvadostest.ActiveUserUUID].Email
+	userUUID := s.users[arvadostest.ActiveUserUUID].UUID
+	groupName := "TestGroup1"
+	// Give read permissions
+	tmpfile, err := MakeTempCSVFile([][]string{{groupName, userEmail, "can_read"}})
+	c.Assert(err, IsNil)
+	defer os.Remove(tmpfile.Name()) // clean up
+	s.cfg.Path = tmpfile.Name()
+	err = doMain(s.cfg)
+	c.Assert(err, IsNil)
+	// Check permissions
+	groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+	c.Assert(err, IsNil)
+	c.Assert(groupUUID, Not(Equals), "")
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, false)
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, false)
+
+	// Give write permissions
+	tmpfile, err = MakeTempCSVFile([][]string{{groupName, userEmail, "can_write"}})
+	c.Assert(err, IsNil)
+	defer os.Remove(tmpfile.Name()) // clean up
+	s.cfg.Path = tmpfile.Name()
+	err = doMain(s.cfg)
+	c.Assert(err, IsNil)
+	// Check permissions
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, false)
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, false)
+
+	// Give manage permissions
+	tmpfile, err = MakeTempCSVFile([][]string{{groupName, userEmail, "can_manage"}})
+	c.Assert(err, IsNil)
+	defer os.Remove(tmpfile.Name()) // clean up
+	s.cfg.Path = tmpfile.Name()
+	err = doMain(s.cfg)
+	c.Assert(err, IsNil)
+	// Check permissions
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_read"), Equals, false)
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_write"), Equals, false)
+	c.Assert(GroupMembershipExists(s.cfg.Client, userUUID, groupUUID, "can_manage"), Equals, true)
+}
+
 // The absence of a user membership on the CSV file implies its removal
 func (s *TestSuite) TestMembershipRemoval(c *C) {
 	localUserEmail := s.users[arvadostest.ActiveUserUUID].Email
@@ -286,8 +379,8 @@ func (s *TestSuite) TestMembershipRemoval(c *C) {
 		groupUUID, err := RemoteGroupExists(s.cfg, groupName)
 		c.Assert(err, IsNil)
 		c.Assert(groupUUID, Not(Equals), "")
-		c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
-		c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
+		c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, true)
+		c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, true)
 	}
 	// New CSV with some previous membership missing
 	data = [][]string{
@@ -304,14 +397,14 @@ func (s *TestSuite) TestMembershipRemoval(c *C) {
 	groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup1")
 	c.Assert(err, IsNil)
 	c.Assert(groupUUID, Not(Equals), "")
-	c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, true)
-	c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, false)
+	c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, false)
 	// Confirm TestGroup1 memberships
 	groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup2")
 	c.Assert(err, IsNil)
 	c.Assert(groupUUID, Not(Equals), "")
-	c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID), Equals, false)
-	c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, localUserUUID, groupUUID, "can_write"), Equals, false)
+	c.Assert(GroupMembershipExists(s.cfg.Client, remoteUserUUID, groupUUID, "can_write"), Equals, true)
 }
 
 // If a group doesn't exist on the system, create it before adding users
@@ -336,7 +429,7 @@ func (s *TestSuite) TestAutoCreateGroupWhenNotExisting(c *C) {
 	c.Assert(err, IsNil)
 	c.Assert(groupUUID, Not(Equals), "")
 	// active user should be a member
-	c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, arvadostest.ActiveUserUUID, groupUUID, "can_write"), Equals, true)
 }
 
 // Users listed on the file that don't exist on the system are ignored
@@ -362,7 +455,7 @@ func (s *TestSuite) TestIgnoreNonexistantUsers(c *C) {
 	groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
 	c.Assert(err, IsNil)
 	c.Assert(groupUUID, Not(Equals), "")
-	c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
 }
 
 // Users listed on the file that don't exist on the system are ignored
@@ -370,13 +463,16 @@ func (s *TestSuite) TestIgnoreEmptyFields(c *C) {
 	activeUserEmail := s.users[arvadostest.ActiveUserUUID].Email
 	activeUserUUID := s.users[arvadostest.ActiveUserUUID].UUID
 	// Confirm that group doesn't exist
-	groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup4")
-	c.Assert(err, IsNil)
-	c.Assert(groupUUID, Equals, "")
+	for _, groupName := range []string{"TestGroup4", "TestGroup5"} {
+		groupUUID, err := RemoteGroupExists(s.cfg, groupName)
+		c.Assert(err, IsNil)
+		c.Assert(groupUUID, Equals, "")
+	}
 	// Create file & run command
 	data := [][]string{
-		{"", activeUserEmail}, // Empty field
-		{"TestGroup5", ""},    // Empty field
+		{"", activeUserEmail},               // Empty field
+		{"TestGroup5", ""},                  // Empty field
+		{"TestGroup5", activeUserEmail, ""}, // Empty 3rd field: is optional but cannot be empty
 		{"TestGroup4", activeUserEmail},
 	}
 	tmpfile, err := MakeTempCSVFile(data)
@@ -385,11 +481,15 @@ func (s *TestSuite) TestIgnoreEmptyFields(c *C) {
 	s.cfg.Path = tmpfile.Name()
 	err = doMain(s.cfg)
 	c.Assert(err, IsNil)
-	// Confirm that memberships exist
+	// Confirm that records about TestGroup5 were skipped
+	groupUUID, err := RemoteGroupExists(s.cfg, "TestGroup5")
+	c.Assert(err, IsNil)
+	c.Assert(groupUUID, Equals, "")
+	// Confirm that membership exists
 	groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup4")
 	c.Assert(err, IsNil)
 	c.Assert(groupUUID, Not(Equals), "")
-	c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
 }
 
 // Instead of emails, use username as identifier
@@ -416,5 +516,5 @@ func (s *TestSuite) TestUseUsernames(c *C) {
 	groupUUID, err = RemoteGroupExists(s.cfg, "TestGroup1")
 	c.Assert(err, IsNil)
 	c.Assert(groupUUID, Not(Equals), "")
-	c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID), Equals, true)
+	c.Assert(GroupMembershipExists(s.cfg.Client, activeUserUUID, groupUUID, "can_write"), Equals, true)
 }