X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/7737b743baed690e98882a1bb3e44d1a76e45030..37ab0eedec5eaf99c27b6b64fd04cc9248081713:/build/run-tests.sh?ds=sidebyside diff --git a/build/run-tests.sh b/build/run-tests.sh index af3426cd73..fff095e6e6 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -81,6 +81,7 @@ lib/controller/railsproxy lib/controller/router lib/controller/rpc lib/crunchstat +lib/crunch-run lib/cloud lib/cloud/azure lib/cloud/cloudtest @@ -89,6 +90,7 @@ lib/dispatchcloud/container lib/dispatchcloud/scheduler lib/dispatchcloud/ssh_executor lib/dispatchcloud/worker +lib/mount lib/service services/api services/arv-git-httpd @@ -104,7 +106,6 @@ services/keep-balance services/login-sync services/nodemanager services/nodemanager_integration -services/crunch-run services/crunch-dispatch-local services/crunch-dispatch-slurm services/ws @@ -392,7 +393,7 @@ checkpidfile() { checkhealth() { svc="$1" - base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") + base=$("${VENVDIR}/bin/python" -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]") url="$base/_health/ping" if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then echo "${url} failed" @@ -520,6 +521,10 @@ setup_ruby_environment() { || fatal 'rvm gemset setup' rvm env + (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2 + bundle="$(which bundle)" + echo "$bundle" + "$bundle" version | grep 2.0.2 || fatal 'install bundler' else # When our "bundle install"s need to install new gems to # satisfy dependencies, we want them to go where "gem install @@ -545,12 +550,14 @@ setup_ruby_environment() { echo "Will install dependencies to $(gem env gemdir)" echo "Will install arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" + bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" + ( + export HOME=$GEMHOME + ("$bundle" version | grep -q 2.0.2) \ + || gem install --user bundler -v 2.0.2 + "$bundle" version | tee /dev/stderr | grep -q 'version 2' + ) || fatal 'install bundler' fi - ( - export HOME=$GEMHOME - ("$(gem env gempath | cut -f1 -d:)/bin/bundle" version | grep 2.0.2) \ - || gem install --user bundler -v 2.0.2 - ) || fatal 'install bundler' } with_test_gemset() { @@ -641,8 +648,13 @@ install_env() { . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. - pip install --no-cache-dir PyYAML future \ - || fatal "pip install PyYAML failed" + ( + set -e + "${VENVDIR}/bin/pip" install PyYAML + "${VENV3DIR}/bin/pip" install PyYAML + cd "$WORKSPACE/sdk/python" + python setup.py install + ) || fatal "installing PyYAML and sdk/python failed" # Preinstall libcloud if using a fork; otherwise nodemanager "pip # install" won't pick it up by default. @@ -668,11 +680,6 @@ Warning: python3 could not be found. Python 3 tests will be skipped. EOF fi - - if ! which bundler >/dev/null - then - gem install --user-install bundler || fatal 'Could not install bundler' - fi } retry() { @@ -738,7 +745,7 @@ do_test() { go_ldflags() { version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev} - echo "-X git.curoverse.com/arvados.git/lib/cmd.version=${version} -X main.version=${version}" + echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}" } do_test_once() { @@ -770,7 +777,7 @@ do_test_once() { else # The above form gets verbose even when testargs is # empty, so use this form in such cases: - go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1" + go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1" fi result=${result:-$?} if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]] @@ -884,11 +891,11 @@ bundle_install_trylocal() { ( set -e echo "(Running bundle install --local. 'could not find package' messages are OK.)" - if ! bundle install --local --no-deployment; then + if ! "$bundle" install --local --no-deployment; then echo "(Running bundle install again, without --local.)" - bundle install --no-deployment + "$bundle" install --no-deployment fi - bundle package --all + "$bundle" package ) } @@ -935,6 +942,7 @@ install_services/login-sync() { install_services/api() { stop_services + check_arvados_config "services/api" cd "$WORKSPACE/services/api" \ && RAILS_ENV=test bundle_install_trylocal \ || return 1 @@ -946,7 +954,7 @@ install_services/api() { # database, so that we can drop it. This assumes the current user # is a postgresql superuser. cd "$WORKSPACE/services/api" \ - && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ + && test_database=$("${VENVDIR}/bin/python" -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null mkdir -p "$WORKSPACE/services/api/tmp/pids" @@ -970,15 +978,16 @@ install_services/api() { && git --git-dir internal.git init \ || return 1 - - (cd "$WORKSPACE/services/api" - export RAILS_ENV=test - if bundle exec rails db:environment:set ; then - bundle exec rake db:drop - fi - bundle exec rake db:setup \ - && bundle exec rake db:fixtures:load - ) + ( + set -e + cd "$WORKSPACE/services/api" + export RAILS_ENV=test + if "$bundle" exec rails db:environment:set ; then + "$bundle" exec rake db:drop + fi + "$bundle" exec rake db:setup + "$bundle" exec rake db:fixtures:load + ) || return 1 } declare -a pythonstuff @@ -1003,7 +1012,7 @@ install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ && mkdir -p tmp/cache \ && RAILS_ENV=test bundle_install_trylocal \ - && RAILS_ENV=test RAILS_GROUPS=assets bundle exec rake npm:install + && RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install } test_doc() { @@ -1013,7 +1022,7 @@ test_doc() { ARVADOS_API_HOST=qr1hi.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! - PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST + PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST ) } @@ -1026,12 +1035,12 @@ test_gofmt() { test_services/api() { rm -f "$WORKSPACE/services/api/git-commit.version" cd "$WORKSPACE/services/api" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS=\'-v -d\' ${testargs[services/api]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake test TESTOPTS=\'-v -d\' ${testargs[services/api]} } test_sdk/ruby() { cd "$WORKSPACE/sdk/ruby" \ - && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]} + && "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/ruby]} } test_sdk/R() { @@ -1044,7 +1053,7 @@ test_sdk/R() { test_sdk/cli() { cd "$WORKSPACE/sdk/cli" \ && mkdir -p /tmp/keep \ - && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]} + && KEEP_LOCAL_STORE=/tmp/keep "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/cli]} } test_sdk/java-v2() { @@ -1053,7 +1062,7 @@ test_sdk/java-v2() { test_services/login-sync() { cd "$WORKSPACE/services/login-sync" \ - && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]} + && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]} } test_services/nodemanager_integration() { @@ -1064,31 +1073,31 @@ test_services/nodemanager_integration() { test_apps/workbench_units() { local TASK="test:units" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} } test_apps/workbench_functionals() { local TASK="test:functionals" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} } test_apps/workbench_integration() { local TASK="test:integration" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} } test_apps/workbench_benchmark() { local TASK="test:benchmark" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_benchmark]} } test_apps/workbench_profile() { local TASK="test:profile" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_profile]} } install_deps() { @@ -1098,6 +1107,7 @@ install_deps() { do_install sdk/cli do_install sdk/perl do_install sdk/python pip + do_install sdk/python pip "${VENV3DIR}/bin/" do_install sdk/ruby do_install services/api do_install services/arv-git-httpd go