16314: Merge branch 'master'

Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom@tomclegg.ca>

diff --combined build/run-tests.sh
index f35ae5bc1323ffc641e3870409b26fb86e96990d,6c697a657ba10c829e86db0f0f030e12fd76eb0c..4d7658933614020311382d231cea2666bbaa51d6
--- 1/build/run-tests.sh
--- 2/build/run-tests.sh
+++ b/build/run-tests.sh
@@@ -162,12 -162,9 +162,12 @@@ temp_preserve
  
  clear_temp() {
      if [[ -z "$temp" ]]; then
 -        # we didn't even get as far as making a temp dir
 +        # we did not even get as far as making a temp dir
          :
      elif [[ -z "$temp_preserve" ]]; then
 +        # Go creates readonly dirs in the module cache, which cause
 +        # "rm -rf" to fail unless we chmod first.
 +        chmod -R u+w "$temp"
          rm -rf "$temp"
      else
          echo "Leaving behind temp dirs in $temp"
@@@ -198,7 -195,7 +198,7 @@@ sanity_checks() 
      ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
          || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
      [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \
-       || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
+         || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
      echo Checking dependencies:
      echo "locale: ${LANG}"
      [[ "$(locale charmap)" = "UTF-8" ]] \
@@@ -376,7 -373,7 +376,7 @@@ if [[ ${skip["sdk/R"]} == 1 && ${skip["
  fi
  
  if [[ $NEED_SDK_R == false ]]; then
-       echo "R SDK not needed, it will not be installed."
+         echo "R SDK not needed, it will not be installed."
  fi
  
  checkpidfile() {
@@@ -417,11 -414,11 +417,11 @@@ start_services() 
      . "$VENVDIR/bin/activate"
      echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...'
      if [[ ! -d "$WORKSPACE/services/api/log" ]]; then
-       mkdir -p "$WORKSPACE/services/api/log"
+         mkdir -p "$WORKSPACE/services/api/log"
      fi
      # Remove empty api.pid file if it exists
      if [[ -f "$WORKSPACE/tmp/api.pid" && ! -s "$WORKSPACE/tmp/api.pid" ]]; then
-       rm -f "$WORKSPACE/tmp/api.pid"
+         rm -f "$WORKSPACE/tmp/api.pid"
      fi
      all_services_stopped=
      fail=1
@@@ -544,12 -541,12 +544,12 @@@ setup_ruby_environment() 
  
          tmpdir_gem_home="$(env - PATH="$PATH" HOME="$GEMHOME" gem env gempath | cut -f1 -d:)"
          PATH="$tmpdir_gem_home/bin:$PATH"
 -        export GEM_PATH="$tmpdir_gem_home"
 +        export GEM_PATH="$tmpdir_gem_home:$(gem env gempath)"
  
          echo "Will install dependencies to $(gem env gemdir)"
 -        echo "Will install arvados gems to $tmpdir_gem_home"
 +        echo "Will install bundler and arvados gems to $tmpdir_gem_home"
          echo "Gem search path is GEM_PATH=$GEM_PATH"
 -        bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle"
 +        bundle="$tmpdir_gem_home/bin/bundle"
          (
              export HOME=$GEMHOME
              bundlers="$(gem list --details bundler)"
@@@ -820,19 -817,19 +820,19 @@@ do_test_once() 
  
  check_arvados_config() {
      if [[ "$1" = "env" ]] ; then
-       return
+         return
      fi
      if [[ -z "$ARVADOS_CONFIG" ]] ; then
-       # Create config file.  The run_test_server script requires PyYAML,
-       # so virtualenv needs to be active.  Downstream steps like
-       # workbench install which require a valid config.yml.
-       if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
-           install_env
-       fi
-       . "$VENVDIR/bin/activate"
+         # Create config file.  The run_test_server script requires PyYAML,
+         # so virtualenv needs to be active.  Downstream steps like
+         # workbench install which require a valid config.yml.
+         if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
+             install_env
+         fi
+         . "$VENVDIR/bin/activate"
          cd "$WORKSPACE"
-       eval $(python sdk/python/tests/run_test_server.py setup_config)
-       deactivate
+         eval $(python sdk/python/tests/run_test_server.py setup_config)
+         deactivate
      fi
  }
  
@@@ -1015,7 -1012,7 +1015,7 @@@ test_doc() 
      (
          set -e
          cd "$WORKSPACE/doc"
-         ARVADOS_API_HOST=qr1hi.arvadosapi.com
+         ARVADOS_API_HOST=pirca.arvadosapi.com
          # Make sure python-epydoc is installed or the next line won't
          # do much good!
          PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST

diff --combined tools/arvbox/bin/arvbox
index 7d45ba17c1dcb57846030d36861051daef4fcd0c,5abaa90e36d1cfc60d286a336fb4551b6e1f5ee6..8f13215bcf56d8a03be5cdd3d3983d47122c9616
--- 1/tools/arvbox/bin/arvbox
--- 2/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@@ -44,6 -44,10 +44,6 @@@ if test -z "$ARVADOS_ROOT" ; the
      ARVADOS_ROOT="$ARVBOX_DATA/arvados"
  fi
  
 -if test -z "$SSO_ROOT" ; then
 -    SSO_ROOT="$ARVBOX_DATA/sso-devise-omniauth-provider"
 -fi
 -
  if test -z "$COMPOSER_ROOT" ; then
      COMPOSER_ROOT="$ARVBOX_DATA/composer"
  fi
@@@ -122,6 -126,7 +122,6 @@@ wait_for_arvbox() 
  docker_run_dev() {
      docker run \
           "--volume=$ARVADOS_ROOT:/usr/src/arvados:rw" \
 -           "--volume=$SSO_ROOT:/usr/src/sso:rw" \
             "--volume=$COMPOSER_ROOT:/usr/src/composer:rw" \
             "--volume=$WORKBENCH2_ROOT:/usr/src/workbench2:rw" \
             "--volume=$PG_DATA:/var/lib/postgresql:rw" \
@@@ -201,7 -206,8 +201,8 @@@ run() 
                --publish=9002:9002
                --publish=25101:25101
                --publish=8001:8001
-               --publish=8002:8002"
+               --publish=8002:8002
+             --publish=45000-45020:45000-45020"
      else
          PUBLIC=""
      fi
@@@ -233,6 -239,9 +234,6 @@@
          if ! test -d "$ARVADOS_ROOT" ; then
              git clone https://git.arvados.org/arvados.git "$ARVADOS_ROOT"
          fi
 -        if ! test -d "$SSO_ROOT" ; then
 -            git clone https://github.com/arvados/sso-devise-omniauth-provider.git "$SSO_ROOT"
 -        fi
          if ! test -d "$COMPOSER_ROOT" ; then
              git clone https://github.com/arvados/composer.git "$COMPOSER_ROOT"
              git -C "$COMPOSER_ROOT" checkout arvados-fork
@@@ -259,6 -268,11 +260,6 @@@
                         /usr/local/lib/arvbox/runsu.sh \
                         /usr/local/lib/arvbox/waitforpostgres.sh
  
 -                docker exec -ti \
 -                       $ARVBOX_CONTAINER \
 -                       /usr/local/lib/arvbox/runsu.sh \
 -                       /var/lib/arvbox/service/sso/run-service --only-setup
 -
                  docker exec -ti \
                         $ARVBOX_CONTAINER \
                         /usr/local/lib/arvbox/runsu.sh \
@@@ -497,7 -511,6 +498,7 @@@ case "$subcmd" i
                      exit 1
                  fi
                  set -x
 +                chmod -R u+w "$ARVBOX_DATA"
                  rm -rf "$ARVBOX_DATA"
              else
                  if test "$1" != -f ; then
@@@ -560,6 -573,7 +561,6 @@@
               "$ARVBOX_BASE/$1/gopath" \
               "$ARVBOX_BASE/$1/Rlibs" \
               "$ARVBOX_BASE/$1/arvados" \
 -             "$ARVBOX_BASE/$1/sso-devise-omniauth-provider" \
               "$ARVBOX_BASE/$1/composer" \
               "$ARVBOX_BASE/$1/workbench2" \
               "$ARVBOX_BASE/$2"