X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/c2c27beb2cb306608523874e23ef66424a10453a..7737b743baed690e98882a1bb3e44d1a76e45030:/build/run-tests.sh?ds=sidebyside diff --git a/build/run-tests.sh b/build/run-tests.sh index ea4d6e4bd2..af3426cd73 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -283,6 +283,10 @@ sanity_checks() { # needed for pkgdown, builds R SDK doc pages which pandoc || fatal "No pandoc. Try: apt-get install pandoc" fi + echo 'procs with /dev/fuse open:' + find /proc/*/fd -lname /dev/fuse 2>/dev/null | cut -d/ -f3 | xargs --no-run-if-empty ps -lywww + echo 'grep fuse /proc/self/mountinfo:' + grep fuse /proc/self/mountinfo } rotate_logfile() { @@ -470,6 +474,7 @@ stop_services() { && python sdk/python/tests/run_test_server.py stop \ && all_services_stopped=1 deactivate + unset ARVADOS_CONFIG } interrupt() { @@ -541,8 +546,11 @@ setup_ruby_environment() { echo "Will install arvados gems to $tmpdir_gem_home" echo "Gem search path is GEM_PATH=$GEM_PATH" fi - bundle config || gem install bundler \ - || fatal 'install bundler' + ( + export HOME=$GEMHOME + ("$(gem env gempath | cut -f1 -d:)/bin/bundle" version | grep 2.0.2) \ + || gem install --user bundler -v 2.0.2 + ) || fatal 'install bundler' } with_test_gemset() { @@ -626,29 +634,8 @@ initialize() { } install_env() { - ( - set -e - mkdir -p "$GOPATH/src/git.curoverse.com" - if [[ ! -h "$GOPATH/src/git.curoverse.com/arvados.git" ]]; then - for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \ - "$GOPATH/src/git.curoverse.com/arvados.git/tmp" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -d "$d" ]] && rmdir "$d" - done - fi - for d in \ - "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \ - "$GOPATH/src/git.curoverse.com/arvados.git"; do - [[ -h "$d" ]] && rm "$d" - done - ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" - go get -v github.com/kardianos/govendor - cd "$GOPATH/src/git.curoverse.com/arvados.git" - go get -v -d ... - "$GOPATH/bin/govendor" sync - which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports - ) || fatal "Go setup failed" + go mod download || fatal "Go deps failed" + which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports || fatal "Go setup failed" setup_virtualenv "$VENVDIR" --python python2.7 . "$VENVDIR/bin/activate" @@ -712,8 +699,6 @@ retry() { } do_test() { - check_arvados_config "$1" - case "${1}" in apps/workbench_units | apps/workbench_functionals | apps/workbench_integration) suite=apps/workbench @@ -733,12 +718,16 @@ do_test() { case "${1}" in services/api) stop_services + check_arvados_config "$1" ;; - gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker) + gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker) + check_arvados_config "$1" # don't care whether services are running ;; *) + check_arvados_config "$1" if ! start_services; then + checkexit 1 "$1 tests" title "test $1 -- failed to start services" return 1 fi @@ -747,6 +736,11 @@ do_test() { retry do_test_once ${@} } +go_ldflags() { + version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev} + echo "-X git.curoverse.com/arvados.git/lib/cmd.version=${version} -X main.version=${version}" +} + do_test_once() { unset result @@ -762,12 +756,12 @@ do_test_once() { then covername="coverage-$(echo "$1" | sed -e 's/\//_/g')" coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp") - # We do "go get -t" here to catch compilation errors + # We do "go install" here to catch compilation errors # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting # compilation errors. - go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ - cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \ + go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" && \ + cd "$WORKSPACE/$1" && \ if [[ -n "${testargs[$1]}" ]] then # "go test -check.vv giturl" doesn't work, but this @@ -830,16 +824,17 @@ check_arvados_config() { install_env fi . "$VENVDIR/bin/activate" + cd "$WORKSPACE" eval $(python sdk/python/tests/run_test_server.py setup_config) deactivate fi } do_install() { - check_arvados_config "$1" if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then return 0 fi + check_arvados_config "$1" retry do_install_once ${@} } @@ -853,7 +848,7 @@ do_install_once() { result=1 elif [[ "$2" == "go" ]] then - go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" + go install -ldflags "$(go_ldflags)" "$WORKSPACE/$1" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -941,7 +936,8 @@ install_services/login-sync() { install_services/api() { stop_services cd "$WORKSPACE/services/api" \ - && RAILS_ENV=test bundle_install_trylocal + && RAILS_ENV=test bundle_install_trylocal \ + || return 1 rm -f config/environments/test.rb cp config/environments/test.rb.example config/environments/test.rb @@ -1027,27 +1023,6 @@ test_gofmt() { [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]] } -test_govendor() { - ( - set -e - cd "$GOPATH/src/git.curoverse.com/arvados.git" - # Remove cached source dirs in workdir. Otherwise, they will - # not qualify as +missing or +external below, and we won't be - # able to detect that they're missing from vendor/vendor.json. - rm -rf vendor/*/ - go get -v -d ... - "$GOPATH/bin/govendor" sync - if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then - echo >&2 "vendor/vendor.json has unused or missing dependencies -- try: - -(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) - -" - return 1 - fi - ) -} - test_services/api() { rm -f "$WORKSPACE/services/api/git-commit.version" cd "$WORKSPACE/services/api" \ @@ -1107,13 +1082,13 @@ test_apps/workbench_integration() { test_apps/workbench_benchmark() { local TASK="test:benchmark" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench_benchmark]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]} } test_apps/workbench_profile() { local TASK="test:profile" cd "$WORKSPACE/apps/workbench" \ - && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS=\'-v -d\' ${testargs[apps/workbench_profile]} + && eval env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]} } install_deps() { @@ -1173,7 +1148,6 @@ test_all() { fi do_test gofmt - do_test govendor do_test doc do_test sdk/ruby do_test sdk/R