X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/a3c1bd10eaad38b1d05b2c7a1c76619776651059..92bdacbec401d5fce3acef1f89e1c60f59340d47:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 74a701d460..ff6ead0fac 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -35,7 +35,7 @@ Options: --short Skip (or scale down) some slow tests. --interactive Set up, then prompt for test/install steps to perform. WORKSPACE=path Arvados source tree to test. -CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required) +CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb" Restrict apiserver tests to the given file sdk/python_test="--test-suite tests.test_keep_locator" @@ -197,10 +197,8 @@ sanity_checks() { [[ -n "${skip[sanity]}" ]] && return 0 ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)" - [[ -n "$CONFIGSRC" ]] \ - || fatal "CONFIGSRC environment not set (see: $0 --help)" - [[ -s "$CONFIGSRC/config.yml" ]] \ - || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" + [[ -z "$CONFIGSRC" ]] || [[ -s "$CONFIGSRC/config.yml" ]] \ + || fatal "CONFIGSRC is $CONFIGSRC but '$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)" echo Checking dependencies: echo "locale: ${LANG}" [[ "$(locale charmap)" = "UTF-8" ]] \ @@ -262,6 +260,9 @@ sanity_checks() { echo -n 'libpq libpq-fe.h: ' find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" + echo -n 'libpam pam_appl.h: ' + find /usr/include -path '*/security/pam_appl.h' | egrep --max-count=1 . \ + || fatal "No libpam pam_appl.h. Try: apt-get install libpam-dev" echo -n 'postgresql: ' psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" echo -n 'phantomjs: ' @@ -393,7 +394,7 @@ checkpidfile() { checkhealth() { svc="$1" - base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") url="$base/_health/ping" if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then echo "${url} failed" @@ -405,7 +406,7 @@ checkdiscoverydoc() { dd="https://${1}/discovery/v1/apis/arvados/v1/rest" if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd" - tail -v $WORKSPACE/services/api/log/test.log + tail -v $WORKSPACE/tmp/railsapi.log return 1 fi echo "${dd} ok" @@ -553,9 +554,15 @@ setup_ruby_environment() { bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" ( export HOME=$GEMHOME - ("$bundle" version | grep -q 2.0.2) \ - || gem install --user bundler -v 2.0.2 - "$bundle" version | grep 2.0.2 + bundlers="$(gem list --details bundler)" + versions=(1.11.0 1.17.3 2.0.2) + for v in ${versions[@]}; do + if ! echo "$bundlers" | fgrep -q "($v)"; then + gem install --user $(for v in ${versions[@]}; do echo bundler:${v}; done) + break + fi + done + "$bundle" version | tee /dev/stderr | grep -q 'version 2' ) || fatal 'install bundler' fi } @@ -590,6 +597,11 @@ setup_virtualenv() { } initialize() { + # If dependencies like ruby, go, etc. are installed in + # /var/lib/arvados -- presumably by "arvados-server install" -- + # then we want to use those versions, instead of whatever happens + # to be installed in /usr. + PATH="/var/lib/arvados/bin:${PATH}" sanity_checks echo "WORKSPACE=$WORKSPACE" @@ -648,8 +660,13 @@ install_env() { . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. - pip install --no-cache-dir PyYAML future httplib2 \ - || fatal "`pip install PyYAML future httplib2` failed" + ( + set -e + "${VENVDIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip" install PyYAML + cd "$WORKSPACE/sdk/python" + python setup.py install + ) || fatal "installing PyYAML and sdk/python failed" # Preinstall libcloud if using a fork; otherwise nodemanager "pip # install" won't pick it up by default. @@ -890,7 +907,7 @@ bundle_install_trylocal() { echo "(Running bundle install again, without --local.)" "$bundle" install --no-deployment fi - "$bundle" package --all + "$bundle" package ) } @@ -937,6 +954,7 @@ install_services/login-sync() { install_services/api() { stop_services + check_arvados_config "services/api" cd "$WORKSPACE/services/api" \ && RAILS_ENV=test bundle_install_trylocal \ || return 1 @@ -948,7 +966,7 @@ install_services/api() { # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ + && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -972,15 +990,16 @@ install_services/api() { && git --git-dir internal.git init \ || return 1 - - (cd "$WORKSPACE/services/api" - export RAILS_ENV=test - if "$bundle" exec rails db:environment:set ; then - "$bundle" exec rake db:drop - fi - "$bundle" exec rake db:setup \ - && "$bundle" exec rake db:fixtures:load - ) + ( + set -e + cd "$WORKSPACE/services/api" + export RAILS_ENV=test + if "$bundle" exec rails db:environment:set ; then + "$bundle" exec rake db:drop + fi + "$bundle" exec rake db:setup + "$bundle" exec rake db:fixtures:load + ) || return 1 } declare -a pythonstuff @@ -1049,7 +1068,7 @@ test_sdk/cli() { } test_sdk/java-v2() { - cd "$WORKSPACE/sdk/java-v2" && gradle test + cd "$WORKSPACE/sdk/java-v2" && gradle test ${testargs[sdk/java-v2]} } test_services/login-sync() { @@ -1099,7 +1118,7 @@ install_deps() { do_install sdk/cli do_install sdk/perl do_install sdk/python pip - do_install sdk/python pip3 + do_install sdk/python pip "${VENV3DIR}/bin/" do_install sdk/ruby do_install services/api do_install services/arv-git-httpd go @@ -1186,6 +1205,7 @@ help_interactive() { echo "== Interactive commands:" echo "TARGET (short for 'test DIR')" echo "test TARGET" + echo "10 test TARGET (run test 10 times)" echo "test TARGET:py3 (test with python3)" echo "test TARGET -check.vv (pass arguments to test)" echo "install TARGET" @@ -1246,6 +1266,10 @@ else while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do history -s "$nextcmd" history -w + count=1 + if [[ "${nextcmd}" =~ ^[0-9] ]]; then + read count nextcmd <<<"${nextcmd}" + fi read verb target opts <<<"${nextcmd}" target="${target%/}" target="${target/\/:/:}" @@ -1265,11 +1289,14 @@ else ${verb}_${target} ;; *) - argstarget=${target%:py3} + argstarget=${target%:py3} testargs["$argstarget"]="${opts}" tt="${testfuncargs[${target}]}" tt="${tt:-$target}" - do_$verb $tt + while [ $count -gt 0 ]; do + do_$verb $tt + let "count=count-1" + done ;; esac ;;