16306: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --combined build/run-tests.sh
index b3eacfe31ed50986ea02223858117f4742d7e474,bedc95b2db28503d2505bc9ce840a06a89662496..610701e6bcf0ab229a4e2988b4c234b84882dd4b
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -91,6 -91,7 +91,7 @@@ lib/dispatchcloud/schedule
  lib/dispatchcloud/ssh_executor
  lib/dispatchcloud/worker
  lib/mount
+ lib/pam
  lib/service
  services/api
  services/arv-git-httpd
@@@ -104,14 -105,10 +105,10 @@@ services/keepprox
  services/keepstore
  services/keep-balance
  services/login-sync
- services/nodemanager
- services/nodemanager_integration
  services/crunch-dispatch-local
  services/crunch-dispatch-slurm
  services/ws
  sdk/cli
- sdk/pam
- sdk/pam:py3
  sdk/python
  sdk/python:py3
  sdk/ruby
@@@ -198,7 -195,7 +195,7 @@@ sanity_checks() 
      ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
          || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
      [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
-       || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
+         || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
      echo Checking dependencies:
      echo "locale: ${LANG}"
      [[ "$(locale charmap)" = "UTF-8" ]] \
@@@ -262,7 -259,7 +259,7 @@@
          || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
      echo -n 'libpam pam_appl.h: '
      find /usr/include -path '*/security/pam_appl.h' | egrep --max-count=1 . \
-         || fatal "No libpam pam_appl.h. Try: apt-get install libpam-dev"
+         || fatal "No libpam pam_appl.h. Try: apt-get install libpam0g-dev"
      echo -n 'postgresql: '
      psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
      echo -n 'phantomjs: '
@@@ -306,8 -303,6 +303,6 @@@ declare -A ski
  declare -A only
  declare -A testargs
  skip[apps/workbench_profile]=1
- # nodemanager_integration tests are not reliable, see #12061.
- skip[services/nodemanager_integration]=1
  
  while [[ -n "$1" ]]
  do
@@@ -378,7 -373,7 +373,7 @@@ if [[ ${skip["sdk/R"]} == 1 && ${skip["
  fi
  
  if [[ $NEED_SDK_R == false ]]; then
-       echo "R SDK not needed, it will not be installed."
+         echo "R SDK not needed, it will not be installed."
  fi
  
  checkpidfile() {
@@@ -419,11 -414,11 +414,11 @@@ start_services() 
      . "$VENVDIR/bin/activate"
      echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
      if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
-       mkdir -p "$WORKSPACE/services/api/log"
+         mkdir -p "$WORKSPACE/services/api/log"
      fi
      # Remove empty api.pid file if it exists
      if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then
-       rm -f "$WORKSPACE/tmp/api.pid"
+         rm -f "$WORKSPACE/tmp/api.pid"
      fi
      all_services_stopped=
      fail=1
@@@ -555,7 -550,7 +550,7 @@@ setup_ruby_environment() 
          (
              export HOME=$GEMHOME
              bundlers="$(gem list --details bundler)"
 -            versions=(1.11.0 1.17.3 2.0.2)
 +            versions=(1.16.6 1.17.3 2.0.2)
              for v in ${versions[@]}; do
                  if ! echo "$bundlers" | fgrep -q "($v)"; then
                      gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done)
@@@ -668,14 -663,6 +663,6 @@@ install_env() 
          python setup.py install
      ) || fatal "installing PyYAML and sdk/python failed"
  
-     # Preinstall libcloud if using a fork; otherwise nodemanager "pip
-     # install" won't pick it up by default.
-     if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then
-         pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
-             || pip install --pre --ignore-installed --no-cache-dir "$LIBCLOUD_PIN_SRC" >/dev/null \
-             || fatal "pip install apache-libcloud failed"
-     fi
- 
      # Deactivate Python 2 virtualenv
      deactivate
  
@@@ -722,9 -709,6 +709,6 @@@ do_test() 
          apps/workbench_units | apps/workbench_functionals | apps/workbench_integration)
              suite=apps/workbench
              ;;
-         services/nodemanager | services/nodemanager_integration)
-             suite=services/nodemanager_suite
-             ;;
          *)
              suite="${1}"
              ;;
@@@ -833,19 -817,19 +817,19 @@@ do_test_once() 
  
  check_arvados_config() {
      if [[ "$1" = "env" ]] ; then
-       return
+         return
      fi
      if [[ -z "$ARVADOS_CONFIG" ]] ; then
-       # Create config file.  The run_test_server script requires PyYAML,
-       # so virtualenv needs to be active.  Downstream steps like
-       # workbench install which require a valid config.yml.
-       if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
-           install_env
-       fi
-       . "$VENVDIR/bin/activate"
+         # Create config file.  The run_test_server script requires PyYAML,
+         # so virtualenv needs to be active.  Downstream steps like
+         # workbench install which require a valid config.yml.
+         if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
+             install_env
+         fi
+         . "$VENVDIR/bin/activate"
          cd "$WORKSPACE"
-       eval $(python sdk/python/tests/run_test_server.py setup_config)
-       deactivate
+         eval $(python sdk/python/tests/run_test_server.py setup_config)
+         deactivate
      fi
  }
  
@@@ -1004,14 -988,12 +988,12 @@@ install_services/api() 
  
  declare -a pythonstuff
  pythonstuff=(
-     sdk/pam
      sdk/python
      sdk/python:py3
      sdk/cwl:py3
      services/dockercleaner:py3
      services/fuse
      services/fuse:py3
-     services/nodemanager
      tools/crunchstat-summary
      tools/crunchstat-summary:py3
  )
@@@ -1076,11 -1058,6 +1058,6 @@@ test_services/login-sync() 
          && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
  }
  
- test_services/nodemanager_integration() {
-     cd "$WORKSPACE/services/nodemanager" \
-         && tests/integration_test.py ${testargs[services/nodemanager_integration]}
- }
- 
  test_apps/workbench_units() {
      local TASK="test:units"
      cd "$WORKSPACE/apps/workbench" \
@@@ -1175,7 -1152,6 +1152,6 @@@ test_all() 
      do_test sdk/cli
      do_test services/login-sync
      do_test sdk/java-v2
-     do_test services/nodemanager_integration
      for p in "${pythonstuff[@]}"
      do
          dir=${p%:py3}

diff --combined doc/_config.yml
index e38f0fa8f7d8afa6fef6f7b6284967234dba68e0,bbab3f307ed88e5eb784c483e692e7173298547a..4fcb00733ee6ff148bfbbcc50370cc905b7f2fad
--- 1/doc/_config.yml
--- 2/doc/_config.yml
+++ b/doc/_config.yml
@@@ -153,8 -153,9 +153,9 @@@ navbar
        - admin/index.html.textile.liquid
      - Users and Groups:
        - admin/user-management.html.textile.liquid
-       - admin/reassign-ownership.html.textile.liquid
        - admin/user-management-cli.html.textile.liquid
+       - admin/reassign-ownership.html.textile.liquid
+       - admin/link-accounts.html.textile.liquid
        - admin/group-management.html.textile.liquid
        - admin/federation.html.textile.liquid
        - admin/merge-remote-account.html.textile.liquid
@@@ -190,8 -191,6 +191,8 @@@
        - install/arvados-on-kubernetes.html.textile.liquid
        - install/arvados-on-kubernetes-minikube.html.textile.liquid
        - install/arvados-on-kubernetes-GKE.html.textile.liquid
 +    - Automatic installation:
 +      - install/automatic.html.textile.liquid
      - Manual installation:
        - install/install-manual-prerequisites.html.textile.liquid
        - install/packages.html.textile.liquid
@@@ -221,9 -220,10 +222,10 @@@
        - install/install-shell-server.html.textile.liquid
        - install/install-webshell.html.textile.liquid
      - Containers API:
-       - install/crunch2-slurm/install-compute-node.html.textile.liquid
        - install/install-jobs-image.html.textile.liquid
-       - install/install-dispatch-cloud.html.textile.liquid
+       - install/crunch2-cloud/install-compute-node.html.textile.liquid
+       - install/crunch2-cloud/install-dispatch-cloud.html.textile.liquid
+       - install/crunch2-slurm/install-compute-node.html.textile.liquid
        - install/crunch2-slurm/install-dispatch.html.textile.liquid
        - install/crunch2-slurm/install-test.html.textile.liquid
      - External dependencies: