lib/dispatchcloud/scheduler
lib/dispatchcloud/ssh_executor
lib/dispatchcloud/worker
+lib/service
services/api
services/arv-git-httpd
services/crunchstat
sdk/go/crunchrunner
sdk/cwl
sdk/R
+sdk/java-v2
tools/sync-groups
tools/crunchstat-summary
+tools/crunchstat-summary:py3
tools/keep-exercise
tools/keep-rsync
tools/keep-block-check
GEMHOME=
PERLINSTALLBASE=
R_LIBS=
+export LANG=en_US.UTF-8
short=
only_install=
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
echo Checking dependencies:
+ echo "locale: ${LANG}"
+ [[ "$(locale charmap)" = "UTF-8" ]] \
+ || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen"
echo -n 'virtualenv: '
virtualenv --version \
|| fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
echo "R SDK not needed, it will not be installed."
fi
+checkpidfile() {
+ svc="$1"
+ pid="$(cat "$WORKSPACE/tmp/${svc}.pid")"
+ if [[ -z "$pid" ]] || ! kill -0 "$pid"; then
+ tail $WORKSPACE/tmp/${1}*.log
+ echo "${svc} pid ${pid} not running"
+ return 1
+ fi
+ echo "${svc} pid ${pid} ok"
+}
+
+checkdiscoverydoc() {
+ dd="https://${1}/discovery/v1/apis/arvados/v1/rest"
+ if ! (set -o pipefail; curl -fsk "$dd" | grep -q ^{ ); then
+ echo >&2 "ERROR: could not retrieve discovery doc from RailsAPI at $dd"
+ tail -v $WORKSPACE/services/api/log/test.log
+ return 1
+ fi
+ echo "${dd} ok"
+}
+
start_services() {
if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
return 0
rm -f "$WORKSPACE/tmp/api.pid"
fi
all_services_stopped=
- fail=0
+ fail=1
cd "$WORKSPACE" \
- && eval $(python sdk/python/tests/run_test_server.py start --auth admin || echo "fail=1; false") \
+ && eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
+ && checkpidfile api \
+ && checkdiscoverydoc $ARVADOS_API_HOST \
&& python sdk/python/tests/run_test_server.py start_controller \
+ && checkpidfile controller \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
+ && checkpidfile keepproxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
+ && checkpidfile keep-web \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+ && checkpidfile arv-git-httpd \
&& python sdk/python/tests/run_test_server.py start_ws \
- && eval $(python sdk/python/tests/run_test_server.py start_nginx || echo "fail=1; false") \
+ && checkpidfile ws \
+ && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
+ && checkdiscoverydoc $ARVADOS_API_HOST \
+ && checkpidfile nginx \
+ && export ARVADOS_TEST_PROXY_SERVICES=1 \
&& (env | egrep ^ARVADOS) \
- || fail=1
+ && fail=0
deactivate
if [[ $fail != 0 ]]; then
unset ARVADOS_TEST_API_HOST
if [[ -n "$all_services_stopped" ]]; then
return
fi
- unset ARVADOS_TEST_API_HOST
- . "$VENVDIR/bin/activate"
+ unset ARVADOS_TEST_API_HOST ARVADOS_TEST_PROXY_SERVICES
+ . "$VENVDIR/bin/activate" || return
cd "$WORKSPACE" \
&& python sdk/python/tests/run_test_server.py stop_nginx \
&& python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
echo "PATH is $PATH"
}
-install_deps() {
+install_env() {
(
set -e
mkdir -p "$GOPATH/src/git.curoverse.com"
services/api)
stop_services
;;
- doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+ gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
# don't care whether services are running
;;
*)
title "test $1"
timer_reset
- if which deactivate >/dev/null; then deactivate; fi; . "$VENVDIR/bin/activate"
- if [[ "$2" == "go" ]]
+ if which deactivate >/dev/null; then deactivate; fi
+ if ! . "$VENVDIR/bin/activate"
+ then
+ result=1
+ elif [[ "$2" == "go" ]]
then
covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
# compilation errors.
go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
- [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
if [[ -n "${testargs[$1]}" ]]
then
# "go test -check.vv giturl" doesn't work, but this
go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
rm "$WORKSPACE/tmp/.$covername.tmp"
fi
+ [[ $result = 0 ]] && gofmt -e -d *.go
elif [[ "$2" == "pip" ]]
then
tries=0
title "install $1"
timer_reset
- if which deactivate >/dev/null; then deactivate; fi; . "$VENVDIR/bin/activate"
- if [[ "$2" == "go" ]]
+ if which deactivate >/dev/null; then deactivate; fi
+ if [[ "$1" != "env" ]] && ! . "$VENVDIR/bin/activate"; then
+ result=1
+ elif [[ "$2" == "go" ]]
then
go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
else
"install_$1"
fi
- result=$?
+ result=${result:-$?}
checkexit $result "$1 install"
title "install $1 -- `timer`"
return $result
# database, so that we can drop it. This assumes the current user
# is a postgresql superuser.
cd "$WORKSPACE/services/api" \
- && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+ && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
&& psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
mkdir -p "$WORKSPACE/services/api/tmp/pids"
lib/dispatchcloud/scheduler
lib/dispatchcloud/ssh_executor
lib/dispatchcloud/worker
+ lib/service
sdk/go/arvados
sdk/go/arvadosclient
sdk/go/auth
)
}
+test_gofmt() {
+ cd "$WORKSPACE" || return 1
+ dirs=$(ls -d */ | egrep -v 'vendor|tmp')
+ [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+}
+
test_services/api() {
rm -f "$WORKSPACE/services/api/git-commit.version"
cd "$WORKSPACE/services/api" \
&& KEEP_LOCAL_STORE=/tmp/keep bundle exec rake test TESTOPTS=-v ${testargs[sdk/cli]}
}
+test_sdk/java-v2() {
+ cd "$WORKSPACE/sdk/java-v2" && gradle test
+}
+
test_services/login-sync() {
cd "$WORKSPACE/services/login-sync" \
&& bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
test_apps/workbench_units() {
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
}
test_apps/workbench_functionals() {
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
}
test_apps/workbench_integration() {
cd "$WORKSPACE/apps/workbench" \
- && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
+ && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
}
test_apps/workbench_benchmark() {
&& env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:profile ${testargs[apps/workbench_profile]}
}
+install_deps() {
+ # Install parts needed by test suites
+ do_install env
+ do_install cmd/arvados-server go
+ do_install sdk/cli
+ do_install sdk/perl
+ do_install sdk/python pip
+ do_install sdk/ruby
+ do_install services/api
+ do_install services/arv-git-httpd go
+ do_install services/keepproxy go
+ do_install services/keepstore go
+ do_install services/keep-web go
+ do_install services/ws go
+}
+
install_all() {
- do_install deps
+ do_install env
do_install doc
do_install sdk/ruby
do_install sdk/R
do_install sdk/perl
do_install sdk/cli
do_install services/login-sync
- # Install the Python SDK early. Various other test suites (like
- # keepproxy) bring up run_test_server.py, which imports the arvados
- # module. We can't actually *test* the Python SDK yet though, because
- # its own test suite brings up some of those other programs (like
- # keepproxy).
for p in "${pythonstuff[@]}"
do
dir=${p%:py3}
exit_cleanly
fi
+ do_test gofmt
do_test doc
do_test sdk/ruby
do_test sdk/R
do_test sdk/cli
do_test services/login-sync
+ do_test sdk/java-v2
do_test services/nodemanager_integration
for p in "${pythonstuff[@]}"
do
help_interactive() {
echo "== Interactive commands:"
- echo "dir (short for 'test dir')"
- echo "test dir"
- echo "test dir:py3 (test with python3)"
- echo "install dir"
- echo "install deps (go/python libs)"
- echo "reset (...services used by integration tests)"
+ echo "TARGET (short for 'test DIR')"
+ echo "test TARGET"
+ echo "test TARGET:py3 (test with python3)"
+ echo "test TARGET -check.vv (pass arguments to test)"
+ echo "install TARGET"
+ echo "install env (go/python libs)"
+ echo "install deps (go/python libs + arvados components needed for integration tests)"
+ echo "reset (...services used by integration tests)"
echo "exit"
echo "== Test targets:"
echo "${!testfuncargs[@]}" | tr ' ' '\n' | sort | column
done
for p in "${pythonstuff[@]}"; do
dir=${p%:py3}
- if [[ ${dir} = ${p} ]]; then
- testfuncargs[$p]="$dir pip $VENVDIR/bin/"
- else
- testfuncargs[$p]="$dir pip $VENV3DIR/bin/"
- fi
+ testfuncargs[$dir]="$dir pip $VENVDIR/bin/"
+ testfuncargs[$dir:py3]="$dir pip $VENV3DIR/bin/"
done
if [[ -z ${interactive} ]]; then
install_all
test_all
else
- stop_services
- verb=test
- target=lib/cmd
+ skip=()
+ only=()
+ only_install=()
+ if [[ -e "$VENVDIR/bin/activate" ]]; then stop_services; fi
+ setnextcmd() {
+ if [[ "$TERM" = dumb ]]; then
+ # assume emacs, or something, is offering a history buffer
+ # and pre-populating the command will only cause trouble
+ nextcmd=
+ elif [[ "$nextcmd" != "install deps" ]]; then
+ :
+ elif [[ -e "$VENVDIR/bin/activate" ]]; then
+ nextcmd="test lib/cmd"
+ else
+ nextcmd="install deps"
+ fi
+ }
echo
help_interactive
- while read -p 'What next? ' -e -i "${verb}${target:+ }${target}" verb target; do
+ nextcmd="install deps"
+ setnextcmd
+ while read -p 'What next? ' -e -i "${nextcmd}" nextcmd; do
+ read verb target opts <<<"${nextcmd}"
+ target="${target%/}"
+ target="${target/\/:/:}"
case "${verb}" in
- "" | "help")
- help_interactive
- ;;
"exit" | "quit")
exit_cleanly
;;
"reset")
stop_services
;;
- *)
- if [[ -z "${target}" ]]; then
- target="${verb}"
- verb=test
- fi
- target="${target%/}"
+ "test" | "install")
case "$target" in
+ "")
+ help_interactive
+ ;;
all | deps)
${verb}_${target}
;;
*)
+ testargs["$target"]="${opts}"
tt="${testfuncargs[${target}]}"
tt="${tt:-$target}"
do_$verb $tt
;;
esac
;;
+ "" | "help" | *)
+ help_interactive
+ ;;
esac
if [[ ${#successes[@]} -gt 0 || ${#failures[@]} -gt 0 ]]; then
report_outcomes
failures=()
fi
cd "$WORKSPACE"
+ setnextcmd
done
echo
fi
projects / arvados.git / blobdiff