X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/3d22ea5b14644b8b66556fb5db5f41a6d84374d5..c0a9b9a3291e14c75711d5849dc5365e2166fccd:/build/run-tests.sh?ds=sidebyside diff --git a/build/run-tests.sh b/build/run-tests.sh index c4c5335596..721269b88e 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -35,7 +35,7 @@ Options: --short Skip (or scale down) some slow tests. --interactive Set up, then prompt for test/install steps to perform. WORKSPACE=path Arvados source tree to test. -CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required) +CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file sdk/python_test="--test-suite tests.test_keep_locator" @@ -90,6 +90,7 @@ lib/dispatchcloud/container lib/dispatchcloud/scheduler lib/dispatchcloud/ssh_executor lib/dispatchcloud/worker +lib/mount lib/service services/api services/arv-git-httpd @@ -126,7 +127,7 @@ sdk/go/blockdigest sdk/go/asyncbuf sdk/go/stats sdk/go/crunchrunner -sdk/cwl +sdk/cwl:py3 sdk/R sdk/java-v2 tools/sync-groups @@ -196,10 +197,8 @@ sanity_checks() { [[ -n "${skip[sanity]}" ]] && return 0 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" - [[ -n "$CONFIGSRC" ]] \ - || fatal "CONFIGSRC environment not set (see: $0 --help)" - [[ -s "$CONFIGSRC/config.yml" ]] \ - || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" + [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \ + || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" echo Checking dependencies: echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ @@ -392,7 +391,7 @@ checkpidfile() { checkhealth() { svc="$1" - base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") url="$base/_health/ping" if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then echo "${url} failed" @@ -404,7 +403,7 @@ checkdiscoverydoc() { dd="https://${1}/discovery/v1/apis/arvados/v1/rest" if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd" - tail -v $WORKSPACE/services/api/log/test.log + tail -v $WORKSPACE/tmp/railsapi.log return 1 fi echo "${dd} ok" @@ -552,9 +551,15 @@ setup_ruby_environment() { bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" ( export HOME=$GEMHOME - ("$bundle" version | grep -q 2.0.2) \ - || gem install --user bundler -v 2.0.2 - "$bundle" version | grep 2.0.2 + bundlers="$(gem list --details bundler)" + versions=(1.11.0 1.17.3 2.0.2) + for v in ${versions[@]}; do + if ! echo "$bundlers" | fgrep -q "($v)"; then + gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) + break + fi + done + "$bundle" version | tee /dev/stderr | grep -q 'version 2' ) || fatal 'install bundler' fi } @@ -589,6 +594,11 @@ setup_virtualenv() { } initialize() { + # If dependencies like ruby, go, etc. are installed in + # /var/lib/arvados -- presumably by "arvados-server install" -- + # then we want to use those versions, instead of whatever happens + # to be installed in /usr. + PATH="/var/lib/arvados/bin:${PATH}" sanity_checks echo "WORKSPACE=$WORKSPACE" @@ -647,8 +657,13 @@ install_env() { . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. - pip install --no-cache-dir PyYAML future httplib2 \ - || fatal "`pip install PyYAML future httplib2` failed" + ( + set -e + "${VENVDIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip" install PyYAML + cd "$WORKSPACE/sdk/python" + python setup.py install + ) || fatal "installing PyYAML and sdk/python failed" # Preinstall libcloud if using a fork; otherwise nodemanager "pip # install" won't pick it up by default. @@ -867,8 +882,8 @@ do_install_once() { cd "$WORKSPACE/$1" \ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ && cd "$WORKSPACE" \ - && "${3}pip" install --no-cache-dir --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --no-cache-dir --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + && "${3}pip" install --no-cache-dir "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --no-cache-dir --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz elif [[ "$2" != "" ]] then "install_$2" @@ -889,7 +904,7 @@ bundle_install_trylocal() { echo "(Running bundle install again, without --local.)" "$bundle" install --no-deployment fi - "$bundle" package --all + "$bundle" package ) } @@ -936,6 +951,7 @@ install_services/login-sync() { install_services/api() { stop_services + check_arvados_config "services/api" cd "$WORKSPACE/services/api" \ && RAILS_ENV=test bundle_install_trylocal \ || return 1 @@ -947,7 +963,7 @@ install_services/api() { # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ + && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -971,15 +987,16 @@ install_services/api() { && git --git-dir internal.git init \ || return 1 - - (cd "$WORKSPACE/services/api" - export RAILS_ENV=test - if "$bundle" exec rails db:environment:set ; then - "$bundle" exec rake db:drop - fi - "$bundle" exec rake db:setup \ - && "$bundle" exec rake db:fixtures:load - ) + ( + set -e + cd "$WORKSPACE/services/api" + export RAILS_ENV=test + if "$bundle" exec rails db:environment:set ; then + "$bundle" exec rake db:drop + fi + "$bundle" exec rake db:setup + "$bundle" exec rake db:fixtures:load + ) || return 1 } declare -a pythonstuff @@ -987,7 +1004,6 @@ pythonstuff=( sdk/pam sdk/python sdk/python:py3 - sdk/cwl sdk/cwl:py3 services/dockercleaner:py3 services/fuse @@ -1049,7 +1065,7 @@ test_sdk/cli() { } test_sdk/java-v2() { - cd "$WORKSPACE/sdk/java-v2" && gradle test + cd "$WORKSPACE/sdk/java-v2" && gradle test ${testargs[sdk/java-v2]} } test_services/login-sync() { @@ -1077,7 +1093,7 @@ test_apps/workbench_functionals() { test_apps/workbench_integration() { local TASK="test:integration" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} } test_apps/workbench_benchmark() { @@ -1099,7 +1115,7 @@ install_deps() { do_install sdk/cli do_install sdk/perl do_install sdk/python pip - do_install sdk/python pip3 + do_install sdk/python pip "${VENV3DIR}/bin/" do_install sdk/ruby do_install services/api do_install services/arv-git-httpd go @@ -1265,7 +1281,8 @@ else ${verb}_${target} ;; *) - testargs["$target"]="${opts}" + argstarget=${target%:py3} + testargs["$argstarget"]="${opts}" tt="${testfuncargs[${target}]}" tt="${tt:-$target}" do_$verb $tt