--skip-install Do not run any install steps. Just run tests.
You should provide GOPATH, GEMHOME, and VENVDIR options
from a previous invocation if you use this option.
+--only-install Run specific install step
WORKSPACE=path Arvados source tree to test.
CONFIGSRC=path Dir with api server config files to copy into source tree.
(If none given, leave config files alone in source tree.)
doc
services/api
services/crunchstat
+services/dockercleaner
services/fuse
services/keepproxy
services/keepstore
GITDIR=
GOPATH=
VENVDIR=
+VENV3DIR=
PYTHONPATH=
GEMHOME=
declare -A leave_temp
clear_temp() {
leaving=""
- for var in VENVDIR GOPATH GITDIR GEMHOME
+ for var in VENVDIR VENV3DIR GOPATH GITDIR GEMHOME
do
if [[ -z "${leave_temp[$var]}" ]]
then
fatal() {
clear_temp
- echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
+ echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
exit 1
}
exit_cleanly() {
trap - INT
+ create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
stop_services
rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
}
sanity_checks() {
- # Make sure WORKSPACE is set
- if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
- fi
-
- # Make sure virtualenv is installed
- `virtualenv --help >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: virtualenv could not be found"
- echo >&2
- exit 1
- fi
-
- # Make sure go is installed
- `go env >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: go could not be found"
- echo >&2
- exit 1
- fi
-
- # Make sure gcc is installed
- `gcc --help >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: gcc could not be found"
- echo >&2
- exit 1
- fi
+ ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+ echo Checking dependencies:
+ echo -n 'virtualenv: '
+ virtualenv --version \
+ || fatal "No virtualenv. Try: apt-get install virtualenv"
+ echo -n 'go: '
+ go version \
+ || fatal "No go binary. See http://golang.org/doc/install"
+ echo -n 'gcc: '
+ gcc --version | egrep ^gcc \
+ || fatal "No gcc. Try: apt-get install build-essential"
+ echo -n 'fuse.h: '
+ find /usr/include -wholename '*fuse/fuse.h' \
+ || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+ echo -n 'pyconfig.h: '
+ find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+ || fatal "No pyconfig.h. Try: apt-get install python-dev"
+ echo -n 'nginx: '
+ PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+ || fatal "No nginx. Try: apt-get install nginx"
}
rotate_logfile() {
+ # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
# $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
if [[ -f "$1/$2" ]]; then
THEDATE=`date +%Y%m%d%H%M%S`
--skip-install)
skip_install=1
;;
+ --only-install)
+ skip_install=1
+ only_install="$1"; shift
+ ;;
--leave-temp)
leave_temp[VENVDIR]=1
+ leave_temp[VENV3DIR]=1
leave_temp[GOPATH]=1
leave_temp[GEMHOME]=1
;;
find -name '*.pyc' -delete
# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR GOPATH GEMHOME
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME
do
if [[ -n "${!tmpdir}" ]]; then
leave_temp[$tmpdir]=1
fi
# Needed for run_test_server.py which is used by certain (non-Python) tests.
-echo "pip install -q PyYAML"
-pip install -q PyYAML || fatal "pip install PyYAML failed"
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+ || pip install PyYAML \
+ || fatal "pip install PyYAML failed"
+
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ virtualenv --python "$PYTHON3" --setuptools "$VENV3DIR" \
+ || fatal "python3 virtualenv $VENV3DIR failed"
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
checkexit() {
if [[ "$1" != "0" ]]; then
fi
elif [[ "$2" == "pip" ]]
then
- cd "$WORKSPACE/$1" \
- && python setup.py test ${testargs[$1]}
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
elif [[ "$2" != "" ]]
then
"test_$2"
}
do_install() {
- if [[ -z "$skip_install" ]]
+ if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
then
title "Running $1 install"
timer_reset
go get -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it belives the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually install the local package
+ # we just built.
cd "$WORKSPACE/$1" \
- && python setup.py sdist rotate --keep=1 --match .tar.gz \
- && pip install -q --upgrade dist/*.tar.gz
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ && cd "$WORKSPACE" \
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
do
do_install "$p" pip
done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
install_apiserver() {
cd "$WORKSPACE/services/api" \
do
do_test "$p" pip
done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
for g in "${gostuff[@]}"
do