--skip-install Do not run any install steps. Just run tests.
You should provide GOPATH, GEMHOME, and VENVDIR options
from a previous invocation if you use this option.
+--only-install Run specific install step
WORKSPACE=path Arvados source tree to test.
CONFIGSRC=path Dir with api server config files to copy into source tree.
(If none given, leave config files alone in source tree.)
Restrict Python SDK tests to the given class
apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
Restrict Workbench tests to the given file
+services/arv-git-httpd_test="-check.vv"
+ Show all log messages, even when tests pass (also works
+ with services/keepstore_test etc.)
ARVADOS_DEBUG=1
Print more debug messages
envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
doc
services/api
services/crunchstat
+services/dockercleaner
services/fuse
services/keepproxy
services/keepstore
GITDIR=
GOPATH=
VENVDIR=
+VENV3DIR=
PYTHONPATH=
GEMHOME=
declare -A leave_temp
clear_temp() {
leaving=""
- for var in VENVDIR GOPATH GITDIR GEMHOME
+ for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
do
if [[ -z "${leave_temp[$var]}" ]]
then
--skip-install)
skip_install=1
;;
+ --only-install)
+ skip_install=1
+ only_install="$1"; shift
+ ;;
--leave-temp)
leave_temp[VENVDIR]=1
+ leave_temp[VENV3DIR]=1
leave_temp[GOPATH]=1
leave_temp[GEMHOME]=1
;;
find -name '*.pyc' -delete
# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR GOPATH GEMHOME
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
do
if [[ -n "${!tmpdir}" ]]; then
leave_temp[$tmpdir]=1
virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
. "$VENVDIR/bin/activate"
+# When re-using $VENVDIR, upgrade any packages (except arvados) that are
+# already installed
+pip install --quiet --upgrade `pip freeze | grep -v arvados | cut -f1 -d=`
+
# Note: this must be the last time we change PATH, otherwise rvm will
# whine a lot.
setup_ruby_environment
# Needed for run_test_server.py which is used by certain (non-Python) tests.
echo "pip install -q PyYAML"
-pip install -q PyYAML || fatal "pip install PyYAML failed"
+pip install --quiet PyYAML || fatal "pip install PyYAML failed"
+
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
+ || fatal "python3 virtualenv $VENV3DIR failed"
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
checkexit() {
if [[ "$1" != "0" ]]; then
timer_reset
if [[ "$2" == "go" ]]
then
- go test ${testargs[$1]} "git.curoverse.com/arvados.git/$1"
+ if [[ -n "${testargs[$1]}" ]]
+ then
+ # "go test -check.vv giturl" doesn't work, but this
+ # does:
+ cd "$WORKSPACE/$1" && go test ${testargs[$1]}
+ else
+ # The above form gets verbose even when testargs is
+ # empty, so use this form in such cases:
+ go test "git.curoverse.com/arvados.git/$1"
+ fi
elif [[ "$2" == "pip" ]]
then
- cd "$WORKSPACE/$1" \
- && python setup.py test ${testargs[$1]}
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
elif [[ "$2" != "" ]]
then
"test_$2"
}
do_install() {
- if [[ -z "$skip_install" ]]
+ if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
then
title "Running $1 install"
timer_reset
go get -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it belives the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually install the local package
+ # we just built.
cd "$WORKSPACE/$1" \
- && python setup.py sdist rotate --keep=1 --match .tar.gz \
- && pip install -q --upgrade dist/*.tar.gz
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ && cd "$WORKSPACE" \
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
do
do_install "$p" pip
done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
install_apiserver() {
cd "$WORKSPACE/services/api" \
do
do_test "$p" pip
done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
for g in "${gostuff[@]}"
do