X-Git-Url: https://git.arvados.org/arvados.git/blobdiff_plain/08d0b1ab43499b7f13462d5e3555d239b4634d22..05c87aaf4572a6420a61e16495420e99ece2cef6:/build/run-tests.sh diff --git a/build/run-tests.sh b/build/run-tests.sh index a37a0f731e..3cf96f3b4a 100755 --- a/build/run-tests.sh +++ b/build/run-tests.sh @@ -77,9 +77,14 @@ doc lib/cli lib/cmd lib/controller +lib/controller/federation +lib/controller/railsproxy +lib/controller/router +lib/controller/rpc lib/crunchstat lib/cloud lib/cloud/azure +lib/cloud/cloudtest lib/dispatchcloud lib/dispatchcloud/container lib/dispatchcloud/scheduler @@ -91,6 +96,7 @@ services/arv-git-httpd services/crunchstat services/dockercleaner services/fuse +services/fuse:py3 services/health services/keep-web services/keepproxy @@ -105,6 +111,7 @@ services/crunch-dispatch-slurm services/ws sdk/cli sdk/pam +sdk/pam:py3 sdk/python sdk/python:py3 sdk/ruby @@ -122,6 +129,7 @@ sdk/go/stats sdk/go/crunchrunner sdk/cwl sdk/R +sdk/java-v2 tools/sync-groups tools/crunchstat-summary tools/crunchstat-summary:py3 @@ -202,8 +210,8 @@ sanity_checks() { echo -n 'go: ' go version \ || fatal "No go binary. See http://golang.org/doc/install" - [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 10 ]] \ - || fatal "Go >= 1.10 required. See http://golang.org/doc/install" + [[ $(go version) =~ go1.([0-9]+) ]] && [[ ${BASH_REMATCH[1]} -ge 12 ]] \ + || fatal "Go >= 1.12 required. See http://golang.org/doc/install" echo -n 'gcc: ' gcc --version | egrep ^gcc \ || fatal "No gcc. Try: apt-get install build-essential" @@ -219,6 +227,8 @@ sanity_checks() { echo -n 'Python3 pyconfig.h: ' find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \ || fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev" + which netstat \ + || fatal "No netstat. Try: apt-get install net-tools" echo -n 'nginx: ' PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \ || fatal "No nginx. Try: apt-get install nginx" @@ -248,12 +258,6 @@ sanity_checks() { echo -n 'libpq libpq-fe.h: ' find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \ || fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev" - echo -n 'services/api/config/database.yml: ' - if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then - fatal "Please provide a database.yml file for the test suite" - else - echo "OK" - fi echo -n 'postgresql: ' psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common" echo -n 'phantomjs: ' @@ -379,6 +383,20 @@ checkpidfile() { echo "${svc} pid ${pid} ok" } +checkhealth() { + svc="$1" + port="$(cat "$WORKSPACE/tmp/${svc}.port")" + scheme=http + if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then + scheme=https + fi + url="$scheme://localhost:${port}/_health/ping" + if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then + echo "${url} failed" + return 1 + fi +} + checkdiscoverydoc() { dd="https://${1}/discovery/v1/apis/arvados/v1/rest" if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then @@ -394,7 +412,7 @@ start_services() { return 0 fi . "$VENVDIR/bin/activate" - echo 'Starting API, keepproxy, keep-web, ws, arv-git-httpd, and nginx ssl proxy...' + echo 'Starting API, controller, keepproxy, keep-web, arv-git-httpd, ws, and nginx ssl proxy...' if [[ ! -d "$WORKSPACE/services/api/log" ]]; then mkdir -p "$WORKSPACE/services/api/log" fi @@ -412,12 +430,15 @@ start_services() { && checkdiscoverydoc $ARVADOS_API_HOST \ && python sdk/python/tests/run_test_server.py start_controller \ && checkpidfile controller \ + && checkhealth controller \ && python sdk/python/tests/run_test_server.py start_keep_proxy \ && checkpidfile keepproxy \ && python sdk/python/tests/run_test_server.py start_keep-web \ && checkpidfile keep-web \ + && checkhealth keep-web \ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \ && checkpidfile arv-git-httpd \ + && checkhealth arv-git-httpd \ && python sdk/python/tests/run_test_server.py start_ws \ && checkpidfile ws \ && eval $(python sdk/python/tests/run_test_server.py start_nginx) \ @@ -551,8 +572,6 @@ setup_virtualenv() { else "$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7' fi - # ubuntu1404 can't seem to install mock via tests_require, but it can do this. - "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0' } initialize() { @@ -592,6 +611,9 @@ initialize() { export R_LIBS export GOPATH + # Make sure our compiled binaries under test override anything + # else that might be in the environment. + export PATH=$GOPATH/bin:$PATH # Jenkins config requires that glob tmp/*.log match something. Ensure # that happens even if we don't end up running services that set up @@ -601,11 +623,30 @@ initialize() { unset http_proxy https_proxy no_proxy - # Note: this must be the last time we change PATH, otherwise rvm will # whine a lot. setup_ruby_environment + if [[ -s "$CONFIGSRC/config.yml" ]] ; then + cp "$CONFIGSRC/config.yml" "$temp/test-config.yml" + export ARVADOS_CONFIG="$temp/test-config.yml" + else + if [[ -s /etc/arvados/config.yml ]] ; then + python > "$temp/test-config.yml" </dev/null; then deactivate; fi if ! . "$VENVDIR/bin/activate" then @@ -761,7 +788,7 @@ do_test_once() { # before trying "go test". Otherwise, coverage-reporting # mode makes Go show the wrong line numbers when reporting # compilation errors. - go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ + go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \ cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \ if [[ -n "${testargs[$1]}" ]] then @@ -788,7 +815,10 @@ do_test_once() { tries=$((${tries}+1)) # $3 can name a path directory for us to use, including trailing # slash; e.g., the bin/ subdirectory of a virtualenv. - "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]} + if [[ -e "${3}activate" ]]; then + . "${3}activate" + fi + python setup.py ${short:+--short-tests-only} test ${testargs[$1]} result=$? if [[ ${tries} < 3 && ${result} == 137 ]] then @@ -821,12 +851,13 @@ do_install_once() { title "install $1" timer_reset + result= if which deactivate >/dev/null; then deactivate; fi if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then result=1 elif [[ "$2" == "go" ]] then - go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" + go get -ldflags "-X git.curoverse.com/arvados.git/lib/cmd.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" elif [[ "$2" == "pip" ]] then # $3 can name a path directory for us to use, including trailing @@ -883,7 +914,7 @@ install_gem() { && cd "$WORKSPACE/$srcpath" \ && bundle_install_trylocal \ && gem build "$gemname.gemspec" \ - && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1) + && with_test_gemset gem install --no-document $(ls -t "$gemname"-*.gem|head -n1) } install_sdk/ruby() { @@ -955,6 +986,7 @@ install_services/api() { || return 1 cd "$WORKSPACE/services/api" \ + && RAILS_ENV=test bundle exec rails db:environment:set \ && RAILS_ENV=test bundle exec rake db:drop \ && RAILS_ENV=test bundle exec rake db:setup \ && RAILS_ENV=test bundle exec rake db:fixtures:load @@ -969,55 +1001,14 @@ pythonstuff=( sdk/cwl:py3 services/dockercleaner:py3 services/fuse + services/fuse:py3 services/nodemanager tools/crunchstat-summary + tools/crunchstat-summary:py3 ) declare -a gostuff -gostuff=( - cmd/arvados-client - cmd/arvados-server - lib/cli - lib/cmd - lib/controller - lib/crunchstat - lib/cloud - lib/cloud/azure - lib/cloud/ec2 - lib/dispatchcloud - lib/dispatchcloud/container - lib/dispatchcloud/scheduler - lib/dispatchcloud/ssh_executor - lib/dispatchcloud/worker - lib/service - sdk/go/arvados - sdk/go/arvadosclient - sdk/go/auth - sdk/go/blockdigest - sdk/go/dispatch - sdk/go/health - sdk/go/httpserver - sdk/go/manifest - sdk/go/asyncbuf - sdk/go/crunchrunner - sdk/go/stats - services/arv-git-httpd - services/crunchstat - services/health - services/keep-web - services/keepstore - sdk/go/keepclient - services/keep-balance - services/keepproxy - services/crunch-dispatch-local - services/crunch-dispatch-slurm - services/crunch-run - services/ws - tools/keep-block-check - tools/keep-exercise - tools/keep-rsync - tools/sync-groups -) +gostuff=($(cd "$WORKSPACE" && git grep -lw func | grep \\.go | sed -e 's/\/[^\/]*$//' | sort -u)) install_apps/workbench() { cd "$WORKSPACE/apps/workbench" \ @@ -1043,10 +1034,31 @@ test_gofmt() { [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]] } +test_govendor() { + ( + set -e + cd "$GOPATH/src/git.curoverse.com/arvados.git" + # Remove cached source dirs in workdir. Otherwise, they will + # not qualify as +missing or +external below, and we won't be + # able to detect that they're missing from vendor/vendor.json. + rm -rf vendor/*/ + go get -v -d ... + "$GOPATH/bin/govendor" sync + if [[ -n $("$GOPATH/bin/govendor" list +unused +missing +external | tee /dev/stderr) ]]; then + echo >&2 "vendor/vendor.json has unused or missing dependencies -- try: + +(export GOPATH=\"${GOPATH}\"; cd \$GOPATH/src/git.curoverse.com/arvados.git && \$GOPATH/bin/govendor add +missing +external && \$GOPATH/bin/govendor remove +unused) + +" + return 1 + fi + ) +} + test_services/api() { rm -f "$WORKSPACE/services/api/git-commit.version" cd "$WORKSPACE/services/api" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS=-v ${testargs[services/api]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test TESTOPTS='-v -d' ${testargs[services/api]} } test_sdk/ruby() { @@ -1067,6 +1079,10 @@ test_sdk/cli() { && KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]} } +test_sdk/java-v2() { + cd "$WORKSPACE/sdk/java-v2" && gradle test +} + test_services/login-sync() { cd "$WORKSPACE/services/login-sync" \ && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]} @@ -1078,28 +1094,48 @@ test_services/nodemanager_integration() { } test_apps/workbench_units() { + local TASK="test:units" + if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_units]}" ]]; then + TASK="test" + fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_units]} } test_apps/workbench_functionals() { + local TASK="test:functionals" + if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_functionals]}" ]]; then + TASK="test" + fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]} } test_apps/workbench_integration() { + local TASK="test:integration" + if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_integration]}" ]]; then + TASK="test" + fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} TESTOPTS='-v -d' ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]} } test_apps/workbench_benchmark() { + local TASK="test:benchmark" + if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_benchmark]}" ]]; then + TASK="test" + fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_benchmark]} } test_apps/workbench_profile() { + local TASK="test:profile" + if [[ -n "${testargs[apps/workbench]}" ]] || [[ -n "${testargs[apps/workbench_profile]}" ]]; then + TASK="test" + fi cd "$WORKSPACE/apps/workbench" \ - && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]} + && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake ${TASK} ${testargs[apps/workbench_profile]} } install_deps() { @@ -1139,11 +1175,11 @@ install_all() { fi fi done - do_install services/api for g in "${gostuff[@]}" do do_install "$g" go done + do_install services/api do_install apps/workbench } @@ -1159,11 +1195,13 @@ test_all() { fi do_test gofmt + do_test govendor do_test doc do_test sdk/ruby do_test sdk/R do_test sdk/cli do_test services/login-sync + do_test sdk/java-v2 do_test services/nodemanager_integration for p in "${pythonstuff[@]}" do @@ -1217,6 +1255,15 @@ for p in "${pythonstuff[@]}"; do testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/" done +testfuncargs["sdk/cli"]="sdk/cli" +testfuncargs["sdk/R"]="sdk/R" +testfuncargs["sdk/java-v2"]="sdk/java-v2" +testfuncargs["apps/workbench_units"]="apps/workbench_units" +testfuncargs["apps/workbench_functionals"]="apps/workbench_functionals" +testfuncargs["apps/workbench_integration"]="apps/workbench_integration" +testfuncargs["apps/workbench_benchmark"]="apps/workbench_benchmark" +testfuncargs["apps/workbench_profile"]="apps/workbench_profile" + if [[ -z ${interactive} ]]; then install_all test_all @@ -1230,19 +1277,21 @@ else # assume emacs, or something, is offering a history buffer # and pre-populating the command will only cause trouble nextcmd= - elif [[ "$nextcmd" != "install deps" ]]; then - : - elif [[ -e "$VENVDIR/bin/activate" ]]; then - nextcmd="test lib/cmd" - else + elif [[ ! -e "$VENVDIR/bin/activate" ]]; then nextcmd="install deps" + else + nextcmd="" fi } echo help_interactive nextcmd="install deps" setnextcmd - while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do + HISTFILE="$WORKSPACE/tmp/.history" + history -r + while read -p 'What next? ' -e -i "$nextcmd" nextcmd; do + history -s "$nextcmd" + history -w read verb target opts <<<"${nextcmd}" target="${target%/}" target="${target/\/:/:}"