--short Skip (or scale down) some slow tests.
--interactive Set up, then prompt for test/install steps to perform.
WORKSPACE=path Arvados source tree to test.
-CONFIGSRC=path Dir with api server config files to copy into source tree.
- (If none given, leave config files alone in source tree.)
+CONFIGSRC=path Dir with config.yml file containing PostgreSQL section for use by tests. (required)
services/api_test="TEST=test/functional/arvados/v1/collections_controller_test.rb"
Restrict apiserver tests to the given file
sdk/python_test="--test-suite tests.test_keep_locator"
services/crunchstat
services/dockercleaner
services/fuse
+services/fuse:py3
services/health
services/keep-web
services/keepproxy
[[ -n "${skip[sanity]}" ]] && return 0
( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
|| fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
+ [[ -n "$CONFIGSRC" ]] \
+ || fatal "CONFIGSRC environment not set (see: $0 --help)"
+ [[ -s "$CONFIGSRC/config.yml" ]] \
+ || fatal "'$CONFIGSRC/config.yml' is empty or not found (see: $0 --help)"
echo Checking dependencies:
echo "locale: ${LANG}"
[[ "$(locale charmap)" = "UTF-8" ]] \
echo -n 'Python3 pyconfig.h: '
find /usr/include -path '*/python3*/pyconfig.h' | egrep --max-count=1 . \
|| fatal "No Python3 pyconfig.h. Try: apt-get install python3-dev"
+ which netstat \
+ || fatal "No netstat. Try: apt-get install net-tools"
echo -n 'nginx: '
PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin" nginx -v \
|| fatal "No nginx. Try: apt-get install nginx"
echo -n 'libpq libpq-fe.h: '
find /usr/include -path '*/postgresql/libpq-fe.h' | egrep --max-count=1 . \
|| fatal "No libpq libpq-fe.h. Try: apt-get install libpq-dev"
- echo -n 'services/api/config/database.yml: '
- if [[ ! -f "$WORKSPACE/services/api/config/database.yml" ]]; then
- fatal "Please provide a database.yml file for the test suite"
- else
- echo "OK"
- fi
echo -n 'postgresql: '
psql --version || fatal "No postgresql. Try: apt-get install postgresql postgresql-client-common"
echo -n 'phantomjs: '
checkhealth() {
svc="$1"
- port="$(cat "$WORKSPACE/tmp/${svc}.port")"
- scheme=http
- if [[ ${svc} =~ -ssl$ || ${svc} = wss ]]; then
- scheme=https
- fi
- url="$scheme://localhost:${port}/_health/ping"
+ base=$(python -c "import yaml; print list(yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['Services']['$1']['InternalURLs'].keys())[0]")
+ url="$base/_health/ping"
if ! curl -Ss -H "Authorization: Bearer e687950a23c3a9bceec28c6223a06c79" "${url}" | tee -a /dev/stderr | grep '"OK"'; then
echo "${url} failed"
return 1
fi
all_services_stopped=
fail=1
+
cd "$WORKSPACE" \
&& eval $(python sdk/python/tests/run_test_server.py start --auth admin) \
&& export ARVADOS_TEST_API_HOST="$ARVADOS_API_HOST" \
&& export ARVADOS_TEST_API_INSTALLED="$$" \
&& checkpidfile api \
&& checkdiscoverydoc $ARVADOS_API_HOST \
+ && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
+ && checkpidfile nginx \
&& python sdk/python/tests/run_test_server.py start_controller \
&& checkpidfile controller \
- && checkhealth controller \
+ && checkhealth Controller \
+ && checkdiscoverydoc $ARVADOS_API_HOST \
&& python sdk/python/tests/run_test_server.py start_keep_proxy \
&& checkpidfile keepproxy \
&& python sdk/python/tests/run_test_server.py start_keep-web \
&& checkpidfile keep-web \
- && checkhealth keep-web \
+ && checkhealth WebDAV \
&& python sdk/python/tests/run_test_server.py start_arv-git-httpd \
&& checkpidfile arv-git-httpd \
- && checkhealth arv-git-httpd \
+ && checkhealth GitHTTP \
&& python sdk/python/tests/run_test_server.py start_ws \
&& checkpidfile ws \
- && eval $(python sdk/python/tests/run_test_server.py start_nginx) \
- && checkdiscoverydoc $ARVADOS_API_HOST \
- && checkpidfile nginx \
&& export ARVADOS_TEST_PROXY_SERVICES=1 \
&& (env | egrep ^ARVADOS) \
&& fail=0
&& python sdk/python/tests/run_test_server.py stop \
&& all_services_stopped=1
deactivate
+ unset ARVADOS_CONFIG
}
interrupt() {
else
"$venvdest/bin/pip" install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
fi
- # ubuntu1404 can't seem to install mock via tests_require, but it can do this.
- "$venvdest/bin/pip" install --no-cache-dir 'mock>=1.0' 'pbr<1.7.0'
}
initialize() {
echo "WORKSPACE=$WORKSPACE"
- if [[ -z "$CONFIGSRC" ]] && [[ -d "$HOME/arvados-api-server" ]]; then
- # Jenkins expects us to use this by default.
- CONFIGSRC="$HOME/arvados-api-server"
- fi
-
# Clean up .pyc files that may exist in the workspace
cd "$WORKSPACE"
find -name '*.pyc' -delete
unset http_proxy https_proxy no_proxy
-
# Note: this must be the last time we change PATH, otherwise rvm will
# whine a lot.
setup_ruby_environment
for d in \
"$GOPATH/src/git.curoverse.com/arvados.git/tmp/GOPATH" \
"$GOPATH/src/git.curoverse.com/arvados.git/tmp" \
+ "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
"$GOPATH/src/git.curoverse.com/arvados.git"; do
+ [[ -h "$d" ]] && rm "$d"
[[ -d "$d" ]] && rmdir "$d"
done
fi
- for d in \
- "$GOPATH/src/git.curoverse.com/arvados.git/arvados" \
- "$GOPATH/src/git.curoverse.com/arvados.git"; do
- [[ -h "$d" ]] && rm "$d"
- done
ln -vsfT "$WORKSPACE" "$GOPATH/src/git.curoverse.com/arvados.git"
go get -v github.com/kardianos/govendor
cd "$GOPATH/src/git.curoverse.com/arvados.git"
go get -v -d ...
"$GOPATH/bin/govendor" sync
+ which goimports >/dev/null || go get golang.org/x/tools/cmd/goimports
) || fatal "Go setup failed"
setup_virtualenv "$VENVDIR" --python python2.7
. "$VENVDIR/bin/activate"
# Needed for run_test_server.py which is used by certain (non-Python) tests.
- pip install --no-cache-dir PyYAML \
+ pip install --no-cache-dir PyYAML future \
|| fatal "pip install PyYAML failed"
# Preinstall libcloud if using a fork; otherwise nodemanager "pip
case "${1}" in
services/api)
stop_services
+ check_arvados_config "$1"
;;
gofmt | govendor | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cloud/cloudtest | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+ check_arvados_config "$1"
# don't care whether services are running
;;
*)
+ check_arvados_config "$1"
if ! start_services; then
title "test $1 -- failed to start services"
return 1
tries=$((${tries}+1))
# $3 can name a path directory for us to use, including trailing
# slash; e.g., the bin/ subdirectory of a virtualenv.
- "${3}python" setup.py ${short:+--short-tests-only} test ${testargs[$1]}
+ if [[ -e "${3}activate" ]]; then
+ . "${3}activate"
+ fi
+ python setup.py ${short:+--short-tests-only} test ${testargs[$1]}
result=$?
if [[ ${tries} < 3 && ${result} == 137 ]]
then
return $result
}
+check_arvados_config() {
+ if [[ "$1" = "env" ]] ; then
+ return
+ fi
+ if [[ -z "$ARVADOS_CONFIG" ]] ; then
+ # Create config file. The run_test_server script requires PyYAML,
+ # so virtualenv needs to be active. Downstream steps like
+ # workbench install which require a valid config.yml.
+ if [[ ! -s "$VENVDIR/bin/activate" ]] ; then
+ install_env
+ fi
+ . "$VENVDIR/bin/activate"
+ cd "$WORKSPACE"
+ eval $(python sdk/python/tests/run_test_server.py setup_config)
+ deactivate
+ fi
+}
+
do_install() {
if [[ -n "${skip[install]}" || ( -n "${only_install}" && "${only_install}" != "${1}" && "${only_install}" != "${2}" ) ]]; then
return 0
fi
+ check_arvados_config "$1"
retry do_install_once ${@}
}
}
install_services/api() {
+ stop_services
cd "$WORKSPACE/services/api" \
&& RAILS_ENV=test bundle_install_trylocal
rm -f config/environments/test.rb
cp config/environments/test.rb.example config/environments/test.rb
- if [ -n "$CONFIGSRC" ]
- then
- for f in database.yml
- do
- cp "$CONFIGSRC/$f" config/ || fatal "$f"
- done
- fi
-
# Clear out any lingering postgresql connections to the test
# database, so that we can drop it. This assumes the current user
# is a postgresql superuser.
cd "$WORKSPACE/services/api" \
- && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
+ && test_database=$(python -c "import yaml; print yaml.safe_load(file('$ARVADOS_CONFIG'))['Clusters']['zzzzz']['PostgreSQL']['Connection']['dbname']") \
&& psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
mkdir -p "$WORKSPACE/services/api/tmp/pids"
&& git --git-dir internal.git init \
|| return 1
- cd "$WORKSPACE/services/api" \
- && RAILS_ENV=test bundle exec rails db:environment:set \
- && RAILS_ENV=test bundle exec rake db:drop \
- && RAILS_ENV=test bundle exec rake db:setup \
- && RAILS_ENV=test bundle exec rake db:fixtures:load
+
+ (cd "$WORKSPACE/services/api"
+ export RAILS_ENV=test
+ if bundle exec rails db:environment:set ; then
+ bundle exec rake db:drop
+ fi
+ bundle exec rake db:setup \
+ && bundle exec rake db:fixtures:load
+ )
}
declare -a pythonstuff
sdk/cwl:py3
services/dockercleaner:py3
services/fuse
+ services/fuse:py3
services/nodemanager
tools/crunchstat-summary
tools/crunchstat-summary:py3
projects / arvados.git / blobdiff