X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/2581a27602bd98fa532926efc02d9401ee095108..9202497767fc6b06444032c43748d14e17351a12:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index 884eda3da1..249783ea28 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -22,10 +22,13 @@ Options: --leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit. Instead, show the path to give as --temp to reuse them in subsequent invocations. +--repeat N Repeat each install/test step until it succeeds N times. +--retry Prompt to retry if an install or test suite fails. --skip-install Do not run any install steps. Just run tests. You should provide GOPATH, GEMHOME, and VENVDIR options from a previous invocation if you use this option. --only-install Run specific install step +--short Skip (or scale down) some slow tests. WORKSPACE=path Arvados source tree to test. CONFIGSRC=path Dir with api server config files to copy into source tree. (If none given, leave config files alone in source tree.) @@ -57,38 +60,51 @@ https://arvados.org/projects/arvados/wiki/Running_tests Available tests: -apps/workbench +apps/workbench (*) +apps/workbench_units (*) +apps/workbench_functionals (*) +apps/workbench_integration (*) apps/workbench_benchmark apps/workbench_profile doc services/api services/arv-git-httpd +services/boot services/crunchstat services/dockercleaner services/fuse services/keep-web services/keepproxy services/keepstore +services/keep-balance services/login-sync services/nodemanager services/crunch-run services/crunch-dispatch-local services/crunch-dispatch-slurm +services/ws sdk/cli sdk/pam sdk/python sdk/ruby +sdk/go/arvados sdk/go/arvadosclient sdk/go/keepclient +sdk/go/httpserver sdk/go/manifest sdk/go/blockdigest sdk/go/streamer +sdk/go/stats sdk/go/crunchrunner sdk/cwl tools/crunchstat-summary +tools/keep-exercise tools/keep-rsync tools/keep-block-check +(*) apps/workbench is shorthand for apps/workbench_units + + apps/workbench_functionals + apps/workbench_integration + EOF # First make sure to remove any ARVADOS_ variables from the calling @@ -105,7 +121,8 @@ PYTHONPATH= GEMHOME= PERLINSTALLBASE= -skip_install= +short= +only_install= temp= temp_preserve= @@ -144,9 +161,17 @@ sanity_checks() { echo -n 'virtualenv: ' virtualenv --version \ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)" + echo -n 'ruby: ' + ruby -v \ + || fatal "No ruby. Install >=2.1.9 (using rbenv, rvm, or source)" + echo -n 'bundler: ' + bundle version \ + || fatal "No bundler. Try: gem install bundler" echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 7 ]] \ + || fatal "Go >= 1.7 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" @@ -197,17 +222,18 @@ do exit 1 ;; --skip) - skipwhat="$1"; shift - skip[$skipwhat]=1 + skip[$1]=1; shift ;; --only) only="$1"; skip[$1]=""; shift ;; + --short) + short=1 + ;; --skip-install) - skip_install=1 + only_install=nothing ;; --only-install) - skip_install=1 only_install="$1"; shift ;; --temp) @@ -217,6 +243,9 @@ do --leave-temp) temp_preserve=1 ;; + --repeat) + repeat=$((${1}+0)); shift + ;; --retry) retry=1 ;; @@ -241,15 +270,18 @@ start_api() { && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ && export ARVADOS_TEST_API_INSTALLED="$$" \ + && python sdk/python/tests/run_test_server.py start_ws \ + && python sdk/python/tests/run_test_server.py start_nginx \ && (env | egrep ^ARVADOS) } start_nginx_proxy_services() { - echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...' + echo 'Starting keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py start_ws \ && python sdk/python/tests/run_test_server.py start_nginx \ && export ARVADOS_TEST_PROXY_SERVICES=1 } @@ -260,12 +292,15 @@ stop_services() { cd "$WORKSPACE" \ && python sdk/python/tests/run_test_server.py stop_nginx \ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \ + && python sdk/python/tests/run_test_server.py stop_ws \ && python sdk/python/tests/run_test_server.py stop_keep-web \ && python sdk/python/tests/run_test_server.py stop_keep_proxy fi if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then unset ARVADOS_TEST_API_HOST cd "$WORKSPACE" \ + && python sdk/python/tests/run_test_server.py stop_nginx \ + && python sdk/python/tests/run_test_server.py stop_ws \ && python sdk/python/tests/run_test_server.py stop fi } @@ -332,9 +367,12 @@ setup_ruby_environment() { # complaint about not being in first place already. rvm use @default 2>/dev/null - # Create (if needed) and switch to an @arvados-tests - # gemset. (Leave the choice of ruby to the caller.) - rvm use @arvados-tests --create \ + # Create (if needed) and switch to an @arvados-tests-* gemset, + # salting the gemset name so it doesn't interfere with + # concurrent builds in other workspaces. Leave the choice of + # ruby to the caller. + gemset="arvados-tests-$(echo -n "${WORKSPACE}" | md5sum | head -c16)" + rvm use "@${gemset}" --create \ || fatal 'rvm gemset setup' rvm env @@ -387,9 +425,9 @@ setup_virtualenv() { fi if [[ $("$venvdest/bin/python" --version 2>&1) =~ \ 3\.[012]\. ]]; then # pip 8.0.0 dropped support for python 3.2, e.g., debian wheezy - "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7,<8' + "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7,<8' else - "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7' + "$venvdest/bin/pip" install 'setuptools>=18.5' 'pip>=7' fi # ubuntu1404 can't seem to install mock via tests_require, but it can do this. "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0' @@ -417,13 +455,10 @@ pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \ || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \ || fatal "pip install apache-libcloud failed" -# This will help people who reuse --temp dirs when we upgrade to llfuse 0.42 -if egrep -q 'llfuse.*>= *0\.42' "$WORKSPACE/services/fuse/setup.py"; then - # Uninstall old llfuse, because services/fuse "pip install" won't - # upgrade it by default. - if pip freeze | egrep '^llfuse==0\.41\.'; then - yes | pip uninstall 'llfuse<0.42' - fi +# Uninstall old llfuse (<1.0), because services/fuse "pip install" +# won't upgrade it by default. +if pip freeze | egrep '^llfuse==0'; then + yes | pip uninstall 'llfuse<1.0' fi # Deactivate Python 2 virtualenv @@ -460,115 +495,161 @@ then fi retry() { - while ! ${@} && [[ "$retry" == 1 ]] + remain="${repeat}" + while : do - read -p 'Try again? [Y/n] ' x - if [[ "$x" != "y" ]] && [[ "$x" != "" ]] - then + if ${@}; then + if [[ "$remain" -gt 1 ]]; then + remain=$((${remain}-1)) + title "Repeating ${remain} more times" + else + break + fi + elif [[ "$retry" == 1 ]]; then + read -p 'Try again? [Y/n] ' x + if [[ "$x" != "y" ]] && [[ "$x" != "" ]] + then + break + fi + else break fi done } do_test() { - retry do_test_once ${@} + case "${1}" in + apps/workbench_units | apps/workbench_functional | apps/workbench_integration) + suite=apps/workbench + ;; + *) + suite="${1}" + ;; + esac + if [[ -z "${skip[$suite]}" && -z "${skip[$1]}" && \ + (-z "${only}" || "${only}" == "${suite}" || \ + "${only}" == "${1}") ]]; then + retry do_test_once ${@} + else + title "Skipping ${1} tests" + fi } do_test_once() { unset result - if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] ) + + title "Running $1 tests" + timer_reset + if [[ "$2" == "go" ]] then - title "Running $1 tests" - timer_reset - if [[ "$2" == "go" ]] + covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" + coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") + gopkgpath="git.curoverse.com/arvados.git/$1" + # We do "go get -t" here to catch compilation errors + # before trying "go test". Otherwise, coverage-reporting + # mode makes Go show the wrong line numbers when reporting + # compilation errors. + cd "$GOPATH/src/${gopkgpath}" || return 1 + go get -t . || return 1 + go generate || return 1 + gofmt -e -d . | egrep . && result=1 + if [[ -n "${testargs[$1]}" ]] + then + # "go test -check.vv giturl" doesn't work, but this + # does: + go test ${short:+-short} ${testargs[$1]} . + else + # The above form gets verbose even when testargs is + # empty, so use this form in such cases: + go test ${short:+-short} ${coverflags[@]} . + fi + result=${result:-$?} + if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]] then - covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" - coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") - # We do "go get -t" here to catch compilation errors - # before trying "go test". Otherwise, coverage-reporting - # mode makes Go show the wrong line numbers when reporting - # compilation errors. - if [[ -n "${testargs[$1]}" ]] - then - # "go test -check.vv giturl" doesn't work, but this - # does: - cd "$WORKSPACE/$1" && \ - go get -t "git.curoverse.com/arvados.git/$1" && \ - go test ${coverflags[@]} ${testargs[$1]} - else - # The above form gets verbose even when testargs is - # empty, so use this form in such cases: - go get -t "git.curoverse.com/arvados.git/$1" && \ - go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1" - fi - result="$?" go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html" rm "$WORKSPACE/tmp/.$covername.tmp" - elif [[ "$2" == "pip" ]] - then + fi + elif [[ "$2" == "pip" ]] + then + tries=0 + cd "$WORKSPACE/$1" && while : + do + tries=$((${tries}+1)) # $3 can name a path directory for us to use, including trailing # slash; e.g., the bin/ subdirectory of a virtualenv. - cd "$WORKSPACE/$1" \ - && "${3}python" setup.py test ${testargs[$1]} - elif [[ "$2" != "" ]] - then - "test_$2" - else - "test_$1" - fi - result=${result:-$?} - checkexit $result "$1 tests" - title "End of $1 tests (`timer`)" - return $result + "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]} + result=$? + if [[ ${tries} < 3 && ${result} == 137 ]] + then + printf '\n*****\n%s tests killed -- retrying\n*****\n\n' "$1" + continue + else + break + fi + done + elif [[ "$2" != "" ]] + then + "test_$2" else - title "Skipping $1 tests" + "test_$1" + fi + if [[ -e "${WORKSPACE}/${1}/package.json" ]]; then + cd "${WORKSPACE}/${1}" && npm test || result=1 fi + result=${result:-$?} + checkexit $result "$1 tests" + title "End of $1 tests (`timer`)" + return $result } do_install() { - retry do_install_once ${@} + if [[ -z "${only_install}" || "${only_install}" == "${1}" ]]; then + retry do_install_once ${@} + else + title "Skipping $1 install" + fi } do_install_once() { - if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]] + title "Running $1 install" + timer_reset + cd "${WORKSPACE}/${1}" || return 1 + if [[ -e "${WORKSPACE}/${1}/package.json" ]]; then + npm install || return 1 + fi + if [[ "$2" == "go" ]] then - title "Running $1 install" - timer_reset - if [[ "$2" == "go" ]] - then - go get -t "git.curoverse.com/arvados.git/$1" - elif [[ "$2" == "pip" ]] - then - # $3 can name a path directory for us to use, including trailing - # slash; e.g., the bin/ subdirectory of a virtualenv. - - # Need to change to a different directory after creating - # the source dist package to avoid a pip bug. - # see https://arvados.org/issues/5766 for details. - - # Also need to install twice, because if it believes the package is - # already installed, pip it won't install it. So the first "pip - # install" ensures that the dependencies are met, the second "pip - # install" ensures that we've actually installed the local package - # we just built. - cd "$WORKSPACE/$1" \ - && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ - && cd "$WORKSPACE" \ - && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ - && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz - elif [[ "$2" != "" ]] - then - "install_$2" - else - "install_$1" - fi - result=$? - checkexit $result "$1 install" - title "End of $1 install (`timer`)" - return $result + go get -d -t "git.curoverse.com/arvados.git/$1" \ + && go generate \ + && go get "git.curoverse.com/arvados.git/$1" + elif [[ "$2" == "pip" ]] + then + # $3 can name a path directory for us to use, including trailing + # slash; e.g., the bin/ subdirectory of a virtualenv. + + # Need to change to a different directory after creating + # the source dist package to avoid a pip bug. + # see https://arvados.org/issues/5766 for details. + + # Also need to install twice, because if it believes the package is + # already installed, pip it won't install it. So the first "pip + # install" ensures that the dependencies are met, the second "pip + # install" ensures that we've actually installed the local package + # we just built. + "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \ + && cd "$WORKSPACE" \ + && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \ + && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz + elif [[ "$2" != "" ]] + then + "install_$2" else - title "Skipping $1 install" + "install_$1" fi + result=$? + checkexit $result "$1 install" + title "End of $1 install (`timer`)" + return $result } bundle_install_trylocal() { @@ -696,26 +777,30 @@ do_install services/api apiserver declare -a gostuff gostuff=( + sdk/go/arvados sdk/go/arvadosclient sdk/go/blockdigest + sdk/go/httpserver sdk/go/manifest sdk/go/streamer sdk/go/crunchrunner + sdk/go/stats + lib/crunchstat services/arv-git-httpd services/crunchstat services/keep-web services/keepstore sdk/go/keepclient + services/keep-balance services/keepproxy - services/datamanager/summary - services/datamanager/collection - services/datamanager/keep - services/datamanager services/crunch-dispatch-local services/crunch-dispatch-slurm services/crunch-run - tools/keep-rsync + services/ws + services/boot tools/keep-block-check + tools/keep-exercise + tools/keep-rsync ) for g in "${gostuff[@]}" do @@ -746,7 +831,7 @@ stop_services test_apiserver() { rm -f "$WORKSPACE/services/api/git-commit.version" cd "$WORKSPACE/services/api" \ - && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[services/api]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS=-v ${testargs[services/api]} } do_test services/api apiserver @@ -789,24 +874,39 @@ do do_test "$g" go done -test_workbench() { +test_workbench_units() { + start_nginx_proxy_services \ + && cd "$WORKSPACE/apps/workbench" \ + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} +} +do_test apps/workbench_units workbench_units + +test_workbench_functionals() { + start_nginx_proxy_services \ + && cd "$WORKSPACE/apps/workbench" \ + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} +} +do_test apps/workbench_functionals workbench_functionals + +test_workbench_integration() { start_nginx_proxy_services \ && cd "$WORKSPACE/apps/workbench" \ - && RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} } -do_test apps/workbench workbench +do_test apps/workbench_integration workbench_integration + test_workbench_benchmark() { start_nginx_proxy_services \ && cd "$WORKSPACE/apps/workbench" \ - && RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]} } do_test apps/workbench_benchmark workbench_benchmark test_workbench_profile() { start_nginx_proxy_services \ && cd "$WORKSPACE/apps/workbench" \ - && RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]} } do_test apps/workbench_profile workbench_profile