#!/bin/bash
+. `dirname "$(readlink -f "$0")"`/libcloud-pin
+
read -rd "\000" helpmessage <<EOF
$(basename $0): Install and test Arvados components.
--skip FOO Do not test the FOO component.
--only FOO Do not test anything except the FOO component.
+--temp DIR Install components and dependencies under DIR instead of
+ making a new temporary directory. Implies --leave-temp.
--leave-temp Do not remove GOPATH, virtualenv, and other temp dirs at exit.
- Instead, show which directories were used this time so they
- can be reused in subsequent invocations.
+ Instead, show the path to give as --temp to reuse them in
+ subsequent invocations.
--skip-install Do not run any install steps. Just run tests.
You should provide GOPATH, GEMHOME, and VENVDIR options
from a previous invocation if you use this option.
+--only-install Run specific install step
WORKSPACE=path Arvados source tree to test.
CONFIGSRC=path Dir with api server config files to copy into source tree.
(If none given, leave config files alone in source tree.)
Restrict Python SDK tests to the given class
apps/workbench_test="TEST=test/integration/pipeline_instances_test.rb"
Restrict Workbench tests to the given file
+services/arv-git-httpd_test="-check.vv"
+ Show all log messages, even when tests pass (also works
+ with services/keepstore_test etc.)
ARVADOS_DEBUG=1
Print more debug messages
envvar=value Set \$envvar to value. Primarily useful for WORKSPACE,
apps/workbench_profile
doc
services/api
+services/arv-git-httpd
services/crunchstat
+services/dockercleaner
services/fuse
+services/keep-web
services/keepproxy
services/keepstore
+services/login-sync
services/nodemanager
-services/arv-git-httpd
sdk/cli
+sdk/pam
sdk/python
sdk/ruby
sdk/go/arvadosclient
sdk/go/keepclient
+sdk/go/manifest
+sdk/go/blockdigest
sdk/go/streamer
+sdk/go/crunchrunner
+tools/crunchstat-summary
+tools/keep-rsync
EOF
GITDIR=
GOPATH=
VENVDIR=
+VENV3DIR=
PYTHONPATH=
GEMHOME=
+PERLINSTALLBASE=
COLUMNS=80
-leave_temp=
skip_install=
+temp=
+temp_preserve=
-declare -A leave_temp
clear_temp() {
- leaving=""
- for var in VENVDIR GOPATH GITDIR GEMHOME
- do
- if [[ -z "${leave_temp[$var]}" ]]
- then
- if [[ -n "${!var}" ]]
- then
- rm -rf "${!var}"
- fi
- else
- leaving+=" $var=\"${!var}\""
- fi
- done
- if [[ -n "$leaving" ]]; then
- echo "Leaving behind temp dirs: $leaving"
+ if [[ -z "$temp" ]]; then
+ # we didn't even get as far as making a temp dir
+ :
+ elif [[ -z "$temp_preserve" ]]; then
+ rm -rf "$temp"
+ else
+ echo "Leaving behind temp dirs in $temp"
fi
}
fatal() {
clear_temp
- echo >&2 "Fatal: $* in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]}"
+ echo >&2 "Fatal: $* (encountered in ${FUNCNAME[1]} at ${BASH_SOURCE[1]} line ${BASH_LINENO[0]})"
exit 1
}
exit_cleanly() {
trap - INT
+ create-plot-data-from-log.sh $BUILD_NUMBER "$WORKSPACE/apps/workbench/log/test.log" "$WORKSPACE/apps/workbench/log/"
rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
- stop_api
+ stop_services
rotate_logfile "$WORKSPACE/services/api/log/" "test.log"
report_outcomes
clear_temp
}
sanity_checks() {
- # Make sure WORKSPACE is set
- if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2 "$helpmessage"
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
- fi
-
- # Make sure virtualenv is installed
- `virtualenv --help >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: virtualenv could not be found"
- echo >&2
- exit 1
- fi
-
- # Make sure go is installed
- `go env >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: go could not be found"
- echo >&2
- exit 1
- fi
-
- # Make sure gcc is installed
- `gcc --help >/dev/null 2>&1`
-
- if [[ "$?" != "0" ]]; then
- echo >&2
- echo >&2 "Error: gcc could not be found"
- echo >&2
- exit 1
- fi
+ ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
+ || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+ echo Checking dependencies:
+ echo -n 'virtualenv: '
+ virtualenv --version \
+ || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"
+ echo -n 'go: '
+ go version \
+ || fatal "No go binary. See http://golang.org/doc/install"
+ echo -n 'gcc: '
+ gcc --version | egrep ^gcc \
+ || fatal "No gcc. Try: apt-get install build-essential"
+ echo -n 'fuse.h: '
+ find /usr/include -wholename '*fuse/fuse.h' \
+ || fatal "No fuse/fuse.h. Try: apt-get install libfuse-dev"
+ echo -n 'pyconfig.h: '
+ find /usr/include -name pyconfig.h | egrep --max-count=1 . \
+ || fatal "No pyconfig.h. Try: apt-get install python-dev"
+ echo -n 'nginx: '
+ PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
+ || fatal "No nginx. Try: apt-get install nginx"
+ echo -n 'perl: '
+ perl -v | grep version \
+ || fatal "No perl. Try: apt-get install perl"
+ for mod in ExtUtils::MakeMaker JSON LWP Net::SSL; do
+ echo -n "perl $mod: "
+ perl -e "use $mod; print \"\$$mod::VERSION\\n\"" \
+ || fatal "No $mod. Try: apt-get install perl-modules libcrypt-ssleay-perl libjson-perl libwww-perl"
+ done
+ echo -n 'gitolite: '
+ which gitolite \
+ || fatal "No gitolite. Try: apt-get install gitolite3"
}
rotate_logfile() {
+ # i.e. rotate_logfile "$WORKSPACE/apps/workbench/log/" "test.log"
# $BUILD_NUMBER is set by Jenkins if this script is being called as part of a Jenkins run
if [[ -f "$1/$2" ]]; then
THEDATE=`date +%Y%m%d%H%M%S`
--skip-install)
skip_install=1
;;
+ --only-install)
+ skip_install=1
+ only_install="$1"; shift
+ ;;
+ --temp)
+ temp="$1"; shift
+ temp_preserve=1
+ ;;
--leave-temp)
- leave_temp[VENVDIR]=1
- leave_temp[GOPATH]=1
- leave_temp[GEMHOME]=1
+ temp_preserve=1
;;
--retry)
retry=1
&& (env | egrep ^ARVADOS)
}
-stop_api() {
+start_nginx_proxy_services() {
+ echo 'Starting keepproxy, keep-web, arv-git-httpd, and nginx ssl proxy...'
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py start_keep_proxy \
+ && python sdk/python/tests/run_test_server.py start_keep-web \
+ && python sdk/python/tests/run_test_server.py start_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py start_nginx \
+ && export ARVADOS_TEST_PROXY_SERVICES=1
+}
+
+stop_services() {
+ if [[ -n "$ARVADOS_TEST_PROXY_SERVICES" ]]; then
+ unset ARVADOS_TEST_PROXY_SERVICES
+ cd "$WORKSPACE" \
+ && python sdk/python/tests/run_test_server.py stop_nginx \
+ && python sdk/python/tests/run_test_server.py stop_arv-git-httpd \
+ && python sdk/python/tests/run_test_server.py stop_keep-web \
+ && python sdk/python/tests/run_test_server.py stop_keep_proxy
+ fi
if [[ -n "$ARVADOS_TEST_API_HOST" ]]; then
unset ARVADOS_TEST_API_HOST
cd "$WORKSPACE" \
cd "$WORKSPACE"
find -name '*.pyc' -delete
+if [[ -z "$temp" ]]; then
+ temp="$(mktemp -d)"
+fi
+
# Set up temporary install dirs (unless existing dirs were supplied)
-for tmpdir in VENVDIR GOPATH GEMHOME
+for tmpdir in VENVDIR VENV3DIR GOPATH GEMHOME PERLINSTALLBASE
do
- if [[ -n "${!tmpdir}" ]]; then
- leave_temp[$tmpdir]=1
- else
- eval $tmpdir=$(mktemp -d)
+ if [[ -z "${!tmpdir}" ]]; then
+ eval "$tmpdir"="$temp/$tmpdir"
+ fi
+ if ! [[ -d "${!tmpdir}" ]]; then
+ mkdir "${!tmpdir}" || fatal "can't create ${!tmpdir} (does $temp exist?)"
fi
done
if [[ "$using_rvm" == true ]]; then
"$@"
else
- GEM_HOME="$tmpdir_gem_home" "$@"
+ GEM_HOME="$tmpdir_gem_home" GEM_PATH="$tmpdir_gem_home" "$@"
fi
}
+gem_uninstall_if_exists() {
+ if gem list "$1\$" | egrep '^\w'; then
+ gem uninstall --force --all --executables "$1"
+ fi
+}
+
+setup_virtualenv() {
+ local venvdest=$1; shift
+ if ! [[ -e "$venvdest/bin/activate" ]] || ! [[ -e "$venvdest/bin/pip" ]]; then
+ virtualenv --setuptools "$@" "$venvdest" || fatal "virtualenv $venvdest failed"
+ fi
+ "$venvdest/bin/pip" install 'setuptools>=18' 'pip>=7'
+ # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
+ "$venvdest/bin/pip" install 'mock>=1.0' 'pbr<1.7.0'
+}
+
+export PERLINSTALLBASE
+export PERLLIB="$PERLINSTALLBASE/lib/perl5:${PERLLIB:+$PERLLIB}"
+
export GOPATH
mkdir -p "$GOPATH/src/git.curoverse.com"
ln -sfn "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git" \
|| fatal "symlink failed"
-virtualenv --setuptools "$VENVDIR" || fatal "virtualenv $VENVDIR failed"
+setup_virtualenv "$VENVDIR" --python python2.7
+. "$VENVDIR/bin/activate"
+
+# Needed for run_test_server.py which is used by certain (non-Python) tests.
+pip freeze 2>/dev/null | egrep ^PyYAML= \
+ || pip install PyYAML >/dev/null \
+ || fatal "pip install PyYAML failed"
+
+# Preinstall forked version of libcloud, because nodemanager "pip install"
+# won't pick it up by default.
+pip freeze 2>/dev/null | egrep ^apache-libcloud==$LIBCLOUD_PIN \
+ || pip install --pre --ignore-installed https://github.com/curoverse/libcloud/archive/apache-libcloud-$LIBCLOUD_PIN.zip >/dev/null \
+ || fatal "pip install apache-libcloud failed"
+
+# Deactivate Python 2 virtualenv
+deactivate
+
+# If Python 3 is available, set up its virtualenv in $VENV3DIR.
+# Otherwise, skip dependent tests.
+PYTHON3=$(which python3)
+if [ "0" = "$?" ]; then
+ setup_virtualenv "$VENV3DIR" --python python3
+else
+ PYTHON3=
+ skip[services/dockercleaner]=1
+ cat >&2 <<EOF
+
+Warning: python3 could not be found
+services/dockercleaner install and tests will be skipped
+
+EOF
+fi
+
+# Reactivate Python 2 virtualenv
. "$VENVDIR/bin/activate"
# Note: this must be the last time we change PATH, otherwise rvm will
gem install --user-install bundler || fatal 'Could not install bundler'
fi
-# Needed for run_test_server.py which is used by certain (non-Python) tests.
-echo "pip install -q PyYAML"
-pip install -q PyYAML || fatal "pip install PyYAML failed"
-
checkexit() {
if [[ "$1" != "0" ]]; then
title "!!!!!! $2 FAILED !!!!!!"
}
do_test_once() {
+ unset result
if [[ -z "${skip[$1]}" ]] && ( [[ -z "$only" ]] || [[ "$only" == "$1" ]] )
then
title "Running $1 tests"
timer_reset
if [[ "$2" == "go" ]]
then
- go test ${testargs[$1]} "git.curoverse.com/arvados.git/$1"
+ covername="coverage-$(echo "$1" | sed -e 's/\//_/g')"
+ coverflags=("-covermode=count" "-coverprofile=$WORKSPACE/tmp/.$covername.tmp")
+ # We do "go get -t" here to catch compilation errors
+ # before trying "go test". Otherwise, coverage-reporting
+ # mode makes Go show the wrong line numbers when reporting
+ # compilation errors.
+ if [[ -n "${testargs[$1]}" ]]
+ then
+ # "go test -check.vv giturl" doesn't work, but this
+ # does:
+ cd "$WORKSPACE/$1" && \
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ go test ${coverflags[@]} ${testargs[$1]}
+ else
+ # The above form gets verbose even when testargs is
+ # empty, so use this form in such cases:
+ go get -t "git.curoverse.com/arvados.git/$1" && \
+ go test ${coverflags[@]} "git.curoverse.com/arvados.git/$1"
+ fi
+ result="$?"
+ go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
+ rm "$WORKSPACE/tmp/.$covername.tmp"
elif [[ "$2" == "pip" ]]
then
- cd "$WORKSPACE/$1" \
- && python setup.py test ${testargs[$1]}
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+ cd "$WORKSPACE/$1" \
+ && "${3}python" setup.py test ${testargs[$1]}
elif [[ "$2" != "" ]]
then
"test_$2"
else
"test_$1"
fi
- result="$?"
+ result=${result:-$?}
checkexit $result "$1 tests"
title "End of $1 tests (`timer`)"
return $result
}
do_install() {
- if [[ -z "$skip_install" ]]
+ if [[ -z "$skip_install" || (-n "$only_install" && "$only_install" == "$1") ]]
then
title "Running $1 install"
timer_reset
go get -t "git.curoverse.com/arvados.git/$1"
elif [[ "$2" == "pip" ]]
then
+ # $3 can name a path directory for us to use, including trailing
+ # slash; e.g., the bin/ subdirectory of a virtualenv.
+
+ # Need to change to a different directory after creating
+ # the source dist package to avoid a pip bug.
+ # see https://arvados.org/issues/5766 for details.
+
+ # Also need to install twice, because if it believes the package is
+ # already installed, pip it won't install it. So the first "pip
+ # install" ensures that the dependencies are met, the second "pip
+ # install" ensures that we've actually installed the local package
+ # we just built.
cd "$WORKSPACE/$1" \
- && python setup.py sdist rotate --keep=1 --match .tar.gz \
- && pip install -q --upgrade dist/*.tar.gz
+ && "${3}python" setup.py sdist rotate --keep=1 --match .tar.gz \
+ && cd "$WORKSPACE" \
+ && "${3}pip" install --quiet "$WORKSPACE/$1/dist"/*.tar.gz \
+ && "${3}pip" install --quiet --no-deps --ignore-installed "$WORKSPACE/$1/dist"/*.tar.gz
elif [[ "$2" != "" ]]
then
"install_$2"
}
do_install doc
-install_ruby_sdk() {
- with_test_gemset gem uninstall --force --all --executables arvados \
- && cd "$WORKSPACE/sdk/ruby" \
+install_gem() {
+ gemname=$1
+ srcpath=$2
+ with_test_gemset gem_uninstall_if_exists "$gemname" \
+ && cd "$WORKSPACE/$srcpath" \
&& bundle_install_trylocal \
- && gem build arvados.gemspec \
- && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-*.gem|head -n1`
+ && gem build "$gemname.gemspec" \
+ && with_test_gemset gem install --no-ri --no-rdoc $(ls -t "$gemname"-*.gem|head -n1)
+}
+
+install_ruby_sdk() {
+ install_gem arvados sdk/ruby
}
do_install sdk/ruby ruby_sdk
+install_perl_sdk() {
+ cd "$WORKSPACE/sdk/perl" \
+ && perl Makefile.PL INSTALL_BASE="$PERLINSTALLBASE" \
+ && make install INSTALLDIRS=perl
+}
+do_install sdk/perl perl_sdk
+
install_cli() {
- with_test_gemset gem uninstall --force --all --executables arvados-cli \
- && cd "$WORKSPACE/sdk/cli" \
- && bundle_install_trylocal \
- && gem build arvados-cli.gemspec \
- && with_test_gemset gem install --no-ri --no-rdoc `ls -t arvados-cli-*.gem|head -n1`
+ install_gem arvados-cli sdk/cli
}
do_install sdk/cli cli
+install_login-sync() {
+ install_gem arvados-login-sync services/login-sync
+}
+do_install services/login-sync login-sync
+
# Install the Python SDK early. Various other test suites (like
# keepproxy) bring up run_test_server.py, which imports the arvados
# module. We can't actually *test* the Python SDK yet though, because
# keepproxy).
declare -a pythonstuff
pythonstuff=(
+ sdk/pam
sdk/python
services/fuse
services/nodemanager
+ tools/crunchstat-summary
)
for p in "${pythonstuff[@]}"
do
do_install "$p" pip
done
+if [ -n "$PYTHON3" ]; then
+ do_install services/dockercleaner pip "$VENV3DIR/bin/"
+fi
install_apiserver() {
cd "$WORKSPACE/services/api" \
# Clear out any lingering postgresql connections to the test
# database, so that we can drop it. This assumes the current user
# is a postgresql superuser.
- test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']")
- psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
+ cd "$WORKSPACE/services/api" \
+ && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+ && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.procpid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle exec rake db:drop \
declare -a gostuff
gostuff=(
+ sdk/go/arvadosclient
+ sdk/go/blockdigest
+ sdk/go/manifest
+ sdk/go/streamer
+ sdk/go/crunchrunner
services/arv-git-httpd
services/crunchstat
+ services/keep-web
services/keepstore
- services/keepproxy
- sdk/go/arvadosclient
sdk/go/keepclient
- sdk/go/streamer
+ services/keepproxy
+ services/datamanager/summary
+ services/datamanager/collection
+ services/datamanager/keep
+ services/datamanager
+ tools/keep-rsync
)
for g in "${gostuff[@]}"
do
ARVADOS_API_HOST=qr1hi.arvadosapi.com
# Make sure python-epydoc is installed or the next line won't
# do much good!
- PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
+ PYTHONPATH=$WORKSPACE/sdk/python/ bundle exec rake linkchecker baseurl=file://$WORKSPACE/doc/.site/ arvados_workbench_host=https://workbench.$ARVADOS_API_HOST arvados_api_host=$ARVADOS_API_HOST
)
}
do_test doc doclinkchecker
-stop_api
+stop_services
test_apiserver() {
cd "$WORKSPACE/services/api" \
}
do_test sdk/cli cli
+test_login-sync() {
+ cd "$WORKSPACE/services/login-sync" \
+ && bundle exec rake test TESTOPTS=-v ${testargs[services/login-sync]}
+}
+do_test services/login-sync login-sync
+
for p in "${pythonstuff[@]}"
do
do_test "$p" pip
done
+do_test services/dockercleaner pip "$VENV3DIR/bin/"
for g in "${gostuff[@]}"
do
done
test_workbench() {
- cd "$WORKSPACE/apps/workbench" \
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
&& RAILS_ENV=test bundle exec rake test TESTOPTS=-v ${testargs[apps/workbench]}
}
do_test apps/workbench workbench
test_workbench_benchmark() {
- cd "$WORKSPACE/apps/workbench" \
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
&& RAILS_ENV=test bundle exec rake test:benchmark ${testargs[apps/workbench_benchmark]}
}
do_test apps/workbench_benchmark workbench_benchmark
test_workbench_profile() {
- cd "$WORKSPACE/apps/workbench" \
+ start_nginx_proxy_services \
+ && cd "$WORKSPACE/apps/workbench" \
&& RAILS_ENV=test bundle exec rake test:profile ${testargs[apps/workbench_profile]}
}
do_test apps/workbench_profile workbench_profile