X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/b326e357c9d906b8b3d536dc2b1f4356b6073a11..dc021c3b57dcdebe464c148d55f9990a74e8246b:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index b19eafa41e..02a6a4044e 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -283,6 +283,10 @@ sanity_checks() { # needed for pkgdown, builds R SDK doc pages which pandoc || fatal "No pandoc. Try: apt-get install pandoc" fi + echo 'procs with /dev/fuse open:' + find /proc/*/fd -lname /dev/fuse 2>/dev/null | cut -d/ -f3 | xargs --no-run-if-empty ps -lywww + echo 'grep fuse /proc/self/mountinfo:' + grep fuse /proc/self/mountinfo } rotate_logfile() { @@ -422,13 +426,6 @@ start_services() { all_services_stopped= fail=1 - # Create config if it hasn't been created already. Normally - # this happens in install_env because there are downstream - # steps like workbench install which require a valid - # config.yml, but when invoked with --skip-install that doesn't - # happen, so make sure to run it here. - eval $(python sdk/python/tests/run_test_server.py setup_config) - cd "$WORKSPACE" \ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \ && export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \ @@ -477,6 +474,7 @@ stop_services() { && python sdk/python/tests/run_test_server.py stop \ && all_services_stopped=1 deactivate + unset ARVADOS_CONFIG } interrupt() { @@ -522,6 +520,10 @@ setup_ruby_environment() { || fatal 'rvm gemset setup' rvm env + (bundle version | grep -q 2.0.2) || gem install bundler -v 2.0.2 + bundle="$(which bundle)" + echo "$bundle" + "$bundle" version | grep 2.0.2 || fatal 'install bundler' else # When our "bundle install"s need to install new gems to # satisfy dependencies, we want them to go where "gem install @@ -547,9 +549,14 @@ setup_ruby_environment() { echo "Will install dependencies to $(gem env gemdir)" echo "Will install arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" + bundle="$(gem env gempath | cut -f1 -d:)/bin/bundle" + ( + export HOME=$GEMHOME + ("$bundle" version | grep -q 2.0.2) \ + || gem install --user bundler -v 2.0.2 + "$bundle" version | grep 2.0.2 + ) || fatal 'install bundler' fi - bundle config || gem install bundler \ - || fatal 'install bundler' } with_test_gemset() { @@ -633,42 +640,16 @@ initialize() { } install_env() { - ( - set -e - mkdir -p "$GOPATH/src/git.curoverse.com" - if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then - for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \ - "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -d "$d" ]] && rmdir "$d" - done - fi - for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -h "$d" ]] && rm "$d" - done - ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" - go get -v github.com/kardianos/govendor - cd "$GOPATH/src/git.curoverse.com/arvados.git" - go get -v -d ... - "$GOPATH/bin/govendor" sync - which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports - ) || fatal "Go setup failed" + go mod download || fatal "Go deps failed" + which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" # Needed for run_test_server.py which is used by certain (non-Python) tests. - pip install --no-cache-dir PyYAML \ + pip install --no-cache-dir PyYAML future \ || fatal "pip install PyYAML failed" - # Create config file. The run_test_server script requires PyYAML, - # so virtualenv needs to be active. Downstream steps like - # workbench install which require a valid config.yml. - eval $(python sdk/python/tests/run_test_server.py setup_config) - # Preinstall libcloud if using a fork; otherwise nodemanager "pip # install" won't pick it up by default. if [[ -n "$LIBCLOUD_PIN_SRC" ]]; then @@ -693,11 +674,6 @@ Warning: python3 could not be found. Python 3 tests will be skipped. EOF fi - - if ! which bundler >/dev/null - then - gem install --user-install bundler || fatal 'Could not install bundler' - fi } retry() { @@ -743,12 +719,16 @@ do_test() { case "${1}" in services/api) stop_services + check_arvados_config "$1" ;; - gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker) + gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker) + check_arvados_config "$1" # don't care whether services are running ;; *) + check_arvados_config "$1" if ! start_services; then + checkexit 1 "$1 tests" title "test $1 -- failed to start services" return 1 fi @@ -757,6 +737,11 @@ do_test() { retry do_test_once ${@} } +go_ldflags() { + version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev} + echo "-X git.arvados.org/arvados.git/lib/cmd.version=${version} -X main.version=${version}" +} + do_test_once() { unset result @@ -772,12 +757,12 @@ do_test_once() { then covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") - # We do "go get -t" here to catch compilation errors + # We do "go install" here to catch compilation errors # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting # compilation errors. - go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ - cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \ + go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" && \ + cd "$WORKSPACE/$1" && \ if [[ -n "${testargs[$1]}" ]] then # "go test -check.vv giturl" doesn't work, but this @@ -786,7 +771,7 @@ do_test_once() { else # The above form gets verbose even when testargs is # empty, so use this form in such cases: - go test ${short:+-short} ${coverflags[@]} "git.curoverse.com/arvados.git/$1" + go test ${short:+-short} ${coverflags[@]} "git.arvados.org/arvados.git/$1" fi result=${result:-$?} if [[ -f "$WORKSPACE/tmp/.$covername.tmp" ]] @@ -828,10 +813,29 @@ do_test_once() { return $result } +check_arvados_config() { + if [[ "$1" = "env" ]] ; then + return + fi + if [[ -z "$ARVADOS_CONFIG" ]] ; then + # Create config file. The run_test_server script requires PyYAML, + # so virtualenv needs to be active. Downstream steps like + # workbench install which require a valid config.yml. + if [[ ! -s "$VENVDIR/bin/activate" ]] ; then + install_env + fi + . "$VENVDIR/bin/activate" + cd "$WORKSPACE" + eval $(python sdk/python/tests/run_test_server.py setup_config) + deactivate + fi +} + do_install() { if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then return 0 fi + check_arvados_config "$1" retry do_install_once ${@} } @@ -845,7 +849,7 @@ do_install_once() { result=1 elif [[ "$2" == "go" ]] then - go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" + go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -881,11 +885,11 @@ bundle_install_trylocal() { ( set -e echo "(Running bundle install --local. 'could not find package' messages are OK.)" - if ! bundle install --local --no-deployment; then + if ! "$bundle" install --local --no-deployment; then echo "(Running bundle install again, without --local.)" - bundle install --no-deployment + "$bundle" install --no-deployment fi - bundle package --all + "$bundle" package --all ) } @@ -933,7 +937,8 @@ install_services/login-sync() { install_services/api() { stop_services cd "$WORKSPACE/services/api" \ - && RAILS_ENV=test bundle_install_trylocal + && RAILS_ENV=test bundle_install_trylocal \ + || return 1 rm -f config/environments/test.rb cp config/environments/test.rb.example config/environments/test.rb @@ -966,11 +971,15 @@ install_services/api() { && git --git-dir internal.git init \ || return 1 - cd "$WORKSPACE/services/api" \ - && RAILS_ENV=test bundle exec rails db:environment:set \ - && RAILS_ENV=test bundle exec rake db:drop \ - && RAILS_ENV=test bundle exec rake db:setup \ - && RAILS_ENV=test bundle exec rake db:fixtures:load + + (cd "$WORKSPACE/services/api" + export RAILS_ENV=test + if "$bundle" exec rails db:environment:set ; then + "$bundle" exec rake db:drop + fi + "$bundle" exec rake db:setup \ + && "$bundle" exec rake db:fixtures:load + ) } declare -a pythonstuff @@ -995,7 +1004,7 @@ install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ && mkdir -p tmp/cache \ && RAILS_ENV=test bundle_install_trylocal \ - && RAILS_ENV=test RAILS_GROUPS=assets bundle exec rake npm:install + && RAILS_ENV=test RAILS_GROUPS=assets "$bundle" exec rake npm:install } test_doc() { @@ -1005,7 +1014,7 @@ test_doc() { ARVADOS_API_HOST=qr1hi.arvadosapi.com # Make sure python-epydoc is installed or the next line won't # do much good! - PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST + PYTHONPATH=$WORKSPACE/sdk/python/ "$bundle" exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST ) } @@ -1015,36 +1024,15 @@ test_gofmt() { [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]] } -test_govendor() { - ( - set -e - cd "$GOPATH/src/git.curoverse.com/arvados.git" - # Remove cached source dirs in workdir. Otherwise, they will - # not qualify as +missing or +external below, and we won't be - # able to detect that they're missing from vendor/vendor.json. - rm -rf vendor/*/ - go get -v -d ... - "$GOPATH/bin/govendor" sync - if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then - echo >&2 "vendor/vendor.json has unused or missing dependencies -- try: - -(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) - -" - return 1 - fi - ) -} - test_services/api() { rm -f "$WORKSPACE/services/api/git-commit.version" cd "$WORKSPACE/services/api" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake test TESTOPTS=\'-v -d\' ${testargs[services/api]} } test_sdk/ruby() { cd "$WORKSPACE/sdk/ruby" \ - && bundle exec rake test TESTOPTS=-v ${testargs[sdk/ruby]} + && "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/ruby]} } test_sdk/R() { @@ -1057,7 +1045,7 @@ test_sdk/R() { test_sdk/cli() { cd "$WORKSPACE/sdk/cli" \ && mkdir -p /tmp/keep \ - && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]} + && KEEP_LOCAL_STORE=/tmp/keep "$bundle" exec rake test TESTOPTS=-v ${testargs[sdk/cli]} } test_sdk/java-v2() { @@ -1066,7 +1054,7 @@ test_sdk/java-v2() { test_services/login-sync() { cd "$WORKSPACE/services/login-sync" \ - && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]} + && "$bundle" exec rake test TESTOPTS=-v ${testargs[services/login-sync]} } test_services/nodemanager_integration() { @@ -1076,47 +1064,32 @@ test_services/nodemanager_integration() { test_apps/workbench_units() { local TASK="test:units" - if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_units]}" ]]; then - TASK="test" - fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} } test_apps/workbench_functionals() { local TASK="test:functionals" - if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_functionals]}" ]]; then - TASK="test" - fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} } test_apps/workbench_integration() { local TASK="test:integration" - if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_integration]}" ]]; then - TASK="test" - fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} } test_apps/workbench_benchmark() { local TASK="test:benchmark" - if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_benchmark]}" ]]; then - TASK="test" - fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_benchmark]} } test_apps/workbench_profile() { local TASK="test:profile" - if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_profile]}" ]]; then - TASK="test" - fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} "$bundle" exec rake ${TASK} ${testargs[apps/workbench_profile]} } install_deps() { @@ -1176,7 +1149,6 @@ test_all() { fi do_test gofmt - do_test govendor do_test doc do_test sdk/ruby do_test sdk/R